From ce96e0e85b96d0a4d0d0d8828b7d4e27b2bb17ef Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Tue, 9 Jul 2024 16:16:29 +0900 Subject: [PATCH 01/22] decoder/stateless/h264: remove PictureData argument from new_picture new_picture is only used to allocate the resources needed for a new frame and doesn't need this argument. --- src/decoder/stateless/h264.rs | 12 +++--------- src/decoder/stateless/h264/dummy.rs | 9 ++------- src/decoder/stateless/h264/vaapi.rs | 7 +------ 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/src/decoder/stateless/h264.rs b/src/decoder/stateless/h264.rs index 1aa69e4a..93bfc403 100644 --- a/src/decoder/stateless/h264.rs +++ b/src/decoder/stateless/h264.rs @@ -84,11 +84,7 @@ pub trait StatelessH264DecoderBackend: fn new_sequence(&mut self, sps: &Rc) -> StatelessBackendResult<()>; /// Called when the decoder determines that a frame or field was found. - fn new_picture( - &mut self, - picture: &PictureData, - timestamp: u64, - ) -> StatelessBackendResult; + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; /// Called when the decoder determines that a second field was found. /// Indicates that the underlying BackendHandle is to be shared between the @@ -96,7 +92,6 @@ pub trait StatelessH264DecoderBackend: /// resource and can thus be presented together as a single frame. fn new_field_picture( &mut self, - picture: &PictureData, timestamp: u64, first_field: &Self::Handle, ) -> StatelessBackendResult; @@ -1154,10 +1149,9 @@ where debug!("Decode picture POC {:?}", pic.pic_order_cnt); let mut backend_pic = if let Some(first_field) = first_field { - self.backend - .new_field_picture(&pic, timestamp, &first_field.1)? + self.backend.new_field_picture(timestamp, &first_field.1)? } else { - self.backend.new_picture(&pic, timestamp)? + self.backend.new_picture(timestamp)? }; self.backend.start_picture( diff --git a/src/decoder/stateless/h264/dummy.rs b/src/decoder/stateless/h264/dummy.rs index 90b31c82..069b18bf 100644 --- a/src/decoder/stateless/h264/dummy.rs +++ b/src/decoder/stateless/h264/dummy.rs @@ -41,12 +41,7 @@ impl StatelessH264DecoderBackend for Backend { Ok(()) } - fn new_field_picture( - &mut self, - _: &PictureData, - _: u64, - _: &Self::Handle, - ) -> StatelessBackendResult<()> { + fn new_field_picture(&mut self, _: u64, _: &Self::Handle) -> StatelessBackendResult<()> { Ok(()) } @@ -68,7 +63,7 @@ impl StatelessH264DecoderBackend for Backend { }) } - fn new_picture(&mut self, _: &PictureData, _: u64) -> StatelessBackendResult<()> { + fn new_picture(&mut self, _: u64) -> StatelessBackendResult<()> { Ok(()) } } diff --git a/src/decoder/stateless/h264/vaapi.rs b/src/decoder/stateless/h264/vaapi.rs index ee1a0a62..a8435445 100644 --- a/src/decoder/stateless/h264/vaapi.rs +++ b/src/decoder/stateless/h264/vaapi.rs @@ -540,11 +540,7 @@ impl StatelessH264DecoderBackend for Vaapi self.process_picture::(picture) } - fn new_picture( - &mut self, - _: &PictureData, - timestamp: u64, - ) -> StatelessBackendResult { + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { let highest_pool = self.highest_pool(); let surface = highest_pool .get_surface() @@ -561,7 +557,6 @@ impl StatelessH264DecoderBackend for Vaapi fn new_field_picture( &mut self, - _: &PictureData, timestamp: u64, first_field: &Self::Handle, ) -> StatelessBackendResult { From da2b33e00cc19b76b2ef8689931bf60e98b9fdb7 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Tue, 9 Jul 2024 16:24:46 +0900 Subject: [PATCH 02/22] decoder/stateless/h264: let the backend decide which resources it needs The decoder code was checking explicitly for output buffers availability before decoding frames, but that requirement is not needed by all backends - for example V4L2 stateless has different requirements. Move this check to the backend by using the new_picture() method to try and allocate all required resources for decoding a frame. --- src/decoder/stateless/h264.rs | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/decoder/stateless/h264.rs b/src/decoder/stateless/h264.rs index 93bfc403..d5646e34 100644 --- a/src/decoder/stateless/h264.rs +++ b/src/decoder/stateless/h264.rs @@ -41,6 +41,7 @@ use crate::codec::h264::picture::Reference; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; use crate::decoder::stateless::PoolLayer; +use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; @@ -50,7 +51,6 @@ use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; @@ -1093,6 +1093,18 @@ where timestamp: u64, slice: &Slice, ) -> Result, DecodeError> { + // Start by securing the backend picture before modifying our state. + let first_field = self.codec.find_first_field(&slice.header)?; + let mut backend_pic = if let Some(first_field) = &first_field { + self.backend.new_field_picture(timestamp, &first_field.1) + } else { + self.backend.new_picture(timestamp) + } + .map_err(|e| match e { + StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), + e => DecodeError::BackendError(e), + })?; + let nalu_hdr = &slice.nalu.header; if nalu_hdr.idr_pic_flag { @@ -1115,17 +1127,6 @@ where return Err(DecodeError::CheckEvents); } - if self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(anyhow!("No pool found"))? - .num_free_frames() - == 0 - { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } - let current_macroblock = match pps.sps.separate_colour_plane_flag { true => CurrentMacroblockTracking::SeparateColorPlane(Default::default()), false => CurrentMacroblockTracking::NonSeparateColorPlane(0), @@ -1137,7 +1138,6 @@ where self.handle_frame_num_gap(&pps.sps, frame_num, timestamp)?; } - let first_field = self.codec.find_first_field(&slice.header)?; let pic = self.init_current_pic( slice, &pps.sps, @@ -1148,12 +1148,6 @@ where debug!("Decode picture POC {:?}", pic.pic_order_cnt); - let mut backend_pic = if let Some(first_field) = first_field { - self.backend.new_field_picture(timestamp, &first_field.1)? - } else { - self.backend.new_picture(timestamp)? - }; - self.backend.start_picture( &mut backend_pic, &pic, From 7ea6b829b0faf234b4615c714c05fe1261b5c29b Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Tue, 9 Jul 2024 16:30:05 +0900 Subject: [PATCH 03/22] decoder/stateless/vp8: let the backend decide which resources it needs The decoder code was checking explicitly for output buffers availability before decoding frames, but that requirement is not needed by all backends - for example V4L2 stateless has different requirements. Move this check to the backend by introducing a new_picture() method that is responsible for allocating the resources required for a picture and returning the expected error if it cannot secure them. --- src/decoder/stateless/vp8.rs | 32 ++++++++----------- src/decoder/stateless/vp8/dummy.rs | 6 +++- src/decoder/stateless/vp8/vaapi.rs | 49 ++++++++++++++++++------------ 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/decoder/stateless/vp8.rs b/src/decoder/stateless/vp8.rs index 2190441e..078a5038 100644 --- a/src/decoder/stateless/vp8.rs +++ b/src/decoder/stateless/vp8.rs @@ -10,8 +10,6 @@ mod vaapi; use std::os::fd::AsFd; use std::os::fd::BorrowedFd; -use anyhow::anyhow; - use crate::codec::vp8::parser::Frame; use crate::codec::vp8::parser::Header; use crate::codec::vp8::parser::MbLfAdjustments; @@ -20,21 +18,20 @@ use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; use crate::decoder::stateless::PoolLayer; +use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - /// Stateless backend methods specific to VP8. pub trait StatelessVp8DecoderBackend: StatelessDecoderBackend + StatelessDecoderBackendPicture @@ -42,6 +39,9 @@ pub trait StatelessVp8DecoderBackend: /// Called when new stream parameters are found. fn new_sequence(&mut self, header: &Header) -> StatelessBackendResult<()>; + /// Called when the decoder determines that a frame or field was found. + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; + /// Called when the decoder wants the backend to finish the decoding /// operations for `picture`. /// @@ -50,14 +50,14 @@ pub trait StatelessVp8DecoderBackend: #[allow(clippy::too_many_arguments)] fn submit_picture( &mut self, - picture: &Header, + picture: Self::Picture, + hdr: &Header, last_ref: &Option, golden_ref: &Option, alt_ref: &Option, bitstream: &[u8], segmentation: &Segmentation, mb_lf_adjust: &MbLfAdjustments, - timestamp: u64, ) -> StatelessBackendResult; } @@ -178,20 +178,15 @@ where { /// Handle a single frame. fn handle_frame(&mut self, frame: Frame, timestamp: u64) -> Result<(), DecodeError> { - if self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? - .num_free_frames() - == 0 - { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } - let show_frame = frame.header.show_frame; + let picture = self.backend.new_picture(timestamp).map_err(|e| match e { + StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), + e => DecodeError::BackendError(e), + })?; + let decoded_handle = self.backend.submit_picture( + picture, &frame.header, &self.codec.last_picture, &self.codec.golden_ref_picture, @@ -199,7 +194,6 @@ where frame.as_ref(), self.codec.parser.segmentation(), self.codec.parser.mb_lf_adjust(), - timestamp, )?; if self.blocking_mode == BlockingMode::Blocking { diff --git a/src/decoder/stateless/vp8/dummy.rs b/src/decoder/stateless/vp8/dummy.rs index 3a3da5a0..1892b279 100644 --- a/src/decoder/stateless/vp8/dummy.rs +++ b/src/decoder/stateless/vp8/dummy.rs @@ -25,8 +25,13 @@ impl StatelessVp8DecoderBackend for Backend { Ok(()) } + fn new_picture(&mut self, _: u64) -> StatelessBackendResult { + Ok(()) + } + fn submit_picture( &mut self, + _: Self::Picture, _: &Header, _: &Option, _: &Option, @@ -34,7 +39,6 @@ impl StatelessVp8DecoderBackend for Backend { _: &[u8], _: &Segmentation, _: &MbLfAdjustments, - _: u64, ) -> StatelessBackendResult { Ok(Handle { handle: Rc::new(RefCell::new(Default::default())), diff --git a/src/decoder/stateless/vp8/vaapi.rs b/src/decoder/stateless/vp8/vaapi.rs index 119fed1d..8922b8d0 100644 --- a/src/decoder/stateless/vp8/vaapi.rs +++ b/src/decoder/stateless/vp8/vaapi.rs @@ -18,6 +18,7 @@ use crate::backend::vaapi::decoder::va_surface_id; use crate::backend::vaapi::decoder::PoolCreationMode; use crate::backend::vaapi::decoder::VaStreamInfo; use crate::backend::vaapi::decoder::VaapiBackend; +use crate::backend::vaapi::decoder::VaapiPicture; use crate::codec::vp8::parser::Header; use crate::codec::vp8::parser::MbLfAdjustments; use crate::codec::vp8::parser::Segmentation; @@ -208,7 +209,7 @@ fn build_slice_param(frame_hdr: &Header, slice_size: usize) -> anyhow::Result
  • StatelessDecoderBackendPicture for VaapiBackend { - type Picture = (); + type Picture = VaapiPicture; } impl StatelessVp8DecoderBackend for VaapiBackend { @@ -216,40 +217,52 @@ impl StatelessVp8DecoderBackend for VaapiB self.new_sequence(header, PoolCreationMode::Highest) } + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { + let highest_pool = self.highest_pool(); + let surface = highest_pool + .get_surface() + .ok_or(StatelessBackendError::OutOfResources)?; + + let metadata = self.metadata_state.get_parsed()?; + + Ok(VaPicture::new( + timestamp, + Rc::clone(&metadata.context), + surface, + )) + } + fn submit_picture( &mut self, - picture: &Header, + mut picture: Self::Picture, + hdr: &Header, last_ref: &Option, golden_ref: &Option, alt_ref: &Option, bitstream: &[u8], segmentation: &Segmentation, mb_lf_adjust: &MbLfAdjustments, - timestamp: u64, ) -> StatelessBackendResult { + let highest_pool = self.highest_pool(); let last_ref = va_surface_id(last_ref); let golden_ref = va_surface_id(golden_ref); let alt_ref = va_surface_id(alt_ref); - let highest_pool = self.highest_pool(); - let surface = highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; let coded_resolution = highest_pool.coded_resolution(); let metadata = self.metadata_state.get_parsed()?; let context = &metadata.context; let iq_buffer = context - .create_buffer(build_iq_matrix(picture, segmentation)?) + .create_buffer(build_iq_matrix(hdr, segmentation)?) .context("while creating IQ matrix buffer")?; let probs = context - .create_buffer(build_probability_table(picture)) + .create_buffer(build_probability_table(hdr)) .context("while creating probability table buffer")?; let pic_param = context .create_buffer(build_pic_param( - picture, + hdr, &coded_resolution, segmentation, mb_lf_adjust, @@ -260,23 +273,21 @@ impl StatelessVp8DecoderBackend for VaapiB .context("while creating pic params buffer")?; let slice_param = context - .create_buffer(build_slice_param(picture, bitstream.len())?) + .create_buffer(build_slice_param(hdr, bitstream.len())?) .context("while creating slice params buffer")?; let slice_data = context .create_buffer(libva::BufferType::SliceData(Vec::from(bitstream))) .context("while creating slice data buffer")?; - let mut va_picture = VaPicture::new(timestamp, Rc::clone(context), surface); - // Add buffers with the parsed data. - va_picture.add_buffer(iq_buffer); - va_picture.add_buffer(probs); - va_picture.add_buffer(pic_param); - va_picture.add_buffer(slice_param); - va_picture.add_buffer(slice_data); + picture.add_buffer(iq_buffer); + picture.add_buffer(probs); + picture.add_buffer(pic_param); + picture.add_buffer(slice_param); + picture.add_buffer(slice_data); - self.process_picture::(va_picture) + self.process_picture::(picture) } } From 87ac422a198b2896b013449592f9d26282960d6e Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:12:49 +0900 Subject: [PATCH 04/22] decoder/stateless/vp9/vaapi: check for resources first in submit_picture --- src/decoder/stateless/vp9/vaapi.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/decoder/stateless/vp9/vaapi.rs b/src/decoder/stateless/vp9/vaapi.rs index 54377c30..ab8d7dde 100644 --- a/src/decoder/stateless/vp9/vaapi.rs +++ b/src/decoder/stateless/vp9/vaapi.rs @@ -253,6 +253,11 @@ impl StatelessVp9DecoderBackend for VaapiB timestamp: u64, segmentation: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult { + let highest_pool = self.highest_pool(); + let surface = highest_pool + .get_surface() + .ok_or(StatelessBackendError::OutOfResources)?; + let reference_frames: [u32; NUM_REF_FRAMES] = reference_frames .iter() .map(va_surface_id) @@ -260,11 +265,6 @@ impl StatelessVp9DecoderBackend for VaapiB .try_into() .unwrap(); - let highest_pool = self.highest_pool(); - let surface = highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; - let metadata = self.metadata_state.get_parsed()?; let context = &metadata.context; From 3b4b05bcf445a9fae08324c2412d9d1f76243d10 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:13:07 +0900 Subject: [PATCH 05/22] decoder/stateless/vp9: check for decoding state only once per superframe handle_frame cannot change the decoding state, so check for it before processing all frames. --- src/decoder/stateless/vp9.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index aaa118ea..7c215597 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -272,13 +272,15 @@ where } } - for frame in frames { - match &mut self.decoding_state { - // Skip input until we get information from the stream. - DecodingState::AwaitingStreamInfo | DecodingState::Reset => (), - // Ask the client to confirm the format before we can process this. - DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), - DecodingState::Decoding => self.handle_frame(&frame, timestamp)?, + match &mut self.decoding_state { + // Skip input until we get information from the stream. + DecodingState::AwaitingStreamInfo | DecodingState::Reset => (), + // Ask the client to confirm the format before we can process this. + DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), + DecodingState::Decoding => { + for frame in frames { + self.handle_frame(&frame, timestamp)?; + } } } From fc07cd38ff288725d016bd60784219691ef1942d Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:24:00 +0900 Subject: [PATCH 06/22] decoder/stateless/vp9: check for available output buffers only if we are decoding There is no reason to not pursue if we are not in a decoding state, so move that check to a more appropriate place. --- src/decoder/stateless/vp9.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index 7c215597..1a08f192 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -227,20 +227,6 @@ where fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { let frames = self.codec.parser.parse_chunk(bitstream)?; - let num_free_frames = self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? - .num_free_frames(); - - if matches!(self.decoding_state, DecodingState::Decoding) && num_free_frames < frames.len() - { - return Err(DecodeError::NotEnoughOutputBuffers( - frames.len() - num_free_frames, - )); - } - // With SVC, the first frame will usually be a key-frame, with // inter-frames carrying the other layers. // @@ -278,6 +264,21 @@ where // Ask the client to confirm the format before we can process this. DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), DecodingState::Decoding => { + let num_free_frames = self + .backend + .frame_pool(PoolLayer::Highest) + .pop() + .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? + .num_free_frames(); + + if matches!(self.decoding_state, DecodingState::Decoding) + && num_free_frames < frames.len() + { + return Err(DecodeError::NotEnoughOutputBuffers( + frames.len() - num_free_frames, + )); + } + for frame in frames { self.handle_frame(&frame, timestamp)?; } From af3257c43f376942ab055d46593eeffb40d0ed35 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:34:39 +0900 Subject: [PATCH 07/22] decoder/stateless/vp9: split backend picture creation and submission This will allow us to allocate all the frames required prior to processing a superframe and make sure the whole superframe can be processed in one go. --- src/decoder/stateless/vp9.rs | 11 ++++++--- src/decoder/stateless/vp9/dummy.rs | 6 ++++- src/decoder/stateless/vp9/vaapi.rs | 39 +++++++++++++++++------------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index 1a08f192..4d6ff5f6 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -46,6 +46,9 @@ pub trait StatelessVp9DecoderBackend: /// Called when new stream parameters are found. fn new_sequence(&mut self, header: &Header) -> StatelessBackendResult<()>; + /// Allocate all resources required to process a new picture. + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; + /// Called when the decoder wants the backend to finish the decoding /// operations for `picture`. /// @@ -53,10 +56,10 @@ pub trait StatelessVp9DecoderBackend: /// and then assign the ownership of the Picture to the Handle. fn submit_picture( &mut self, - picture: &Header, + picture: Self::Picture, + hdr: &Header, reference_frames: &[Option; NUM_REF_FRAMES], bitstream: &[u8], - timestamp: u64, segmentation: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult; } @@ -173,11 +176,13 @@ where let refresh_frame_flags = frame.header.refresh_frame_flags; Segmentation::update_segmentation(&mut self.codec.segmentation, &frame.header); + let picture = self.backend.new_picture(timestamp)?; + let decoded_handle = self.backend.submit_picture( + picture, &frame.header, &self.codec.reference_frames, frame.as_ref(), - timestamp, &self.codec.segmentation, )?; diff --git a/src/decoder/stateless/vp9/dummy.rs b/src/decoder/stateless/vp9/dummy.rs index a884635c..d39ce14a 100644 --- a/src/decoder/stateless/vp9/dummy.rs +++ b/src/decoder/stateless/vp9/dummy.rs @@ -26,12 +26,16 @@ impl StatelessVp9DecoderBackend for Backend { Ok(()) } + fn new_picture(&mut self, _: u64) -> StatelessBackendResult { + Ok(()) + } + fn submit_picture( &mut self, + _: Self::Picture, _: &Header, _: &[Option; NUM_REF_FRAMES], _: &[u8], - _: u64, _: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult { Ok(Handle { diff --git a/src/decoder/stateless/vp9/vaapi.rs b/src/decoder/stateless/vp9/vaapi.rs index ab8d7dde..603dfe2b 100644 --- a/src/decoder/stateless/vp9/vaapi.rs +++ b/src/decoder/stateless/vp9/vaapi.rs @@ -15,6 +15,7 @@ use crate::backend::vaapi::decoder::va_surface_id; use crate::backend::vaapi::decoder::PoolCreationMode; use crate::backend::vaapi::decoder::VaStreamInfo; use crate::backend::vaapi::decoder::VaapiBackend; +use crate::backend::vaapi::decoder::VaapiPicture; use crate::codec::vp9::parser::BitDepth; use crate::codec::vp9::parser::Header; use crate::codec::vp9::parser::Profile; @@ -237,7 +238,7 @@ fn build_slice_param( } impl StatelessDecoderBackendPicture for VaapiBackend { - type Picture = (); + type Picture = VaapiPicture; } impl StatelessVp9DecoderBackend for VaapiBackend { @@ -245,18 +246,27 @@ impl StatelessVp9DecoderBackend for VaapiB self.new_sequence(header, PoolCreationMode::Highest) } + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { + let highest_pool = self.highest_pool(); + let surface = highest_pool + .get_surface() + .ok_or(StatelessBackendError::OutOfResources)?; + let metadata = self.metadata_state.get_parsed()?; + let context = &metadata.context; + + Ok(VaPicture::new(timestamp, Rc::clone(context), surface)) + } + fn submit_picture( &mut self, - picture: &Header, + mut picture: Self::Picture, + hdr: &Header, reference_frames: &[Option; NUM_REF_FRAMES], bitstream: &[u8], - timestamp: u64, segmentation: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult { - let highest_pool = self.highest_pool(); - let surface = highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + let metadata = self.metadata_state.get_parsed()?; + let context = &metadata.context; let reference_frames: [u32; NUM_REF_FRAMES] = reference_frames .iter() @@ -265,11 +275,8 @@ impl StatelessVp9DecoderBackend for VaapiB .try_into() .unwrap(); - let metadata = self.metadata_state.get_parsed()?; - let context = &metadata.context; - let pic_param = context - .create_buffer(build_pic_param(picture, reference_frames)?) + .create_buffer(build_pic_param(hdr, reference_frames)?) .context("while creating pic params buffer")?; let slice_param = context @@ -280,14 +287,12 @@ impl StatelessVp9DecoderBackend for VaapiB .create_buffer(libva::BufferType::SliceData(Vec::from(bitstream))) .context("while creating slice data buffer")?; - let mut va_picture = VaPicture::new(timestamp, Rc::clone(context), surface); - // Add buffers with the parsed data. - va_picture.add_buffer(pic_param); - va_picture.add_buffer(slice_param); - va_picture.add_buffer(slice_data); + picture.add_buffer(pic_param); + picture.add_buffer(slice_param); + picture.add_buffer(slice_data); - self.process_picture::(va_picture) + self.process_picture::(picture) } } From 18db9dfa95fc34a6138a445cf9bc399c457d1fa0 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:43:56 +0900 Subject: [PATCH 08/22] decoder/stateless/vp9: split display of existing frame into its own method This is a different procedure from decoding an regular frame, which justifies having both separate. This will also let us pass the picture to decode into to `handle_frame` without having to worry whether it will be written or not. --- src/decoder/stateless/vp9.rs | 95 +++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index 4d6ff5f6..bfd3fb25 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -153,55 +153,54 @@ where Ok(()) } - /// Handle a single frame. + /// Handle a frame which `show_existing_frame` flag is `true`. + fn handle_show_existing_frame(&mut self, frame_to_show_map_idx: u8) -> Result<(), DecodeError> { + // Frame to be shown. Because the spec mandates that frame_to_show_map_idx references a + // valid entry in the DPB, an non-existing index means that the stream is invalid. + let idx = usize::from(frame_to_show_map_idx); + let ref_frame = self + .codec + .reference_frames + .get(idx) + .ok_or_else(|| anyhow::anyhow!("invalid reference frame index in header"))? + .as_ref() + .ok_or_else(|| anyhow::anyhow!("empty reference frame referenced in frame header"))?; + + // We are done, no further processing needed. + let decoded_handle = ref_frame.clone(); + + self.ready_queue.push(decoded_handle); + + Ok(()) + } + + /// Decode a single frame. fn handle_frame(&mut self, frame: &Frame, timestamp: u64) -> Result<(), DecodeError> { - let decoded_handle = if frame.header.show_existing_frame { - // Frame to be shown. Because the spec mandates that frame_to_show_map_idx references a - // valid entry in the DPB, an non-existing index means that the stream is invalid. - let idx = usize::from(frame.header.frame_to_show_map_idx); - let ref_frame = self - .codec - .reference_frames - .get(idx) - .ok_or_else(|| anyhow::anyhow!("invalid reference frame index in header"))? - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!("empty reference frame referenced in frame header") - })?; - - // We are done, no further processing needed. - ref_frame.clone() - } else { - // Otherwise, we must actually arrange to decode a frame - let refresh_frame_flags = frame.header.refresh_frame_flags; - - Segmentation::update_segmentation(&mut self.codec.segmentation, &frame.header); - let picture = self.backend.new_picture(timestamp)?; - - let decoded_handle = self.backend.submit_picture( - picture, - &frame.header, - &self.codec.reference_frames, - frame.as_ref(), - &self.codec.segmentation, - )?; - - if self.blocking_mode == BlockingMode::Blocking { - decoded_handle.sync()?; - } + let refresh_frame_flags = frame.header.refresh_frame_flags; + + Segmentation::update_segmentation(&mut self.codec.segmentation, &frame.header); + let picture = self.backend.new_picture(timestamp)?; - // Do DPB management - Self::update_references( - &mut self.codec.reference_frames, - &decoded_handle, - refresh_frame_flags, - )?; + let decoded_handle = self.backend.submit_picture( + picture, + &frame.header, + &self.codec.reference_frames, + frame.as_ref(), + &self.codec.segmentation, + )?; + + if self.blocking_mode == BlockingMode::Blocking { + decoded_handle.sync()?; + } - decoded_handle - }; + // Do DPB management + Self::update_references( + &mut self.codec.reference_frames, + &decoded_handle, + refresh_frame_flags, + )?; - let show_existing_frame = frame.header.show_existing_frame; - if frame.header.show_frame || show_existing_frame { + if frame.header.show_frame { self.ready_queue.push(decoded_handle); } @@ -285,7 +284,11 @@ where } for frame in frames { - self.handle_frame(&frame, timestamp)?; + if frame.header.show_existing_frame { + self.handle_show_existing_frame(frame.header.frame_to_show_map_idx)?; + } else { + self.handle_frame(&frame, timestamp)?; + } } } } From e6e1e752887211efe7cd1f2839a43e940cde5135 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:46:27 +0900 Subject: [PATCH 09/22] decoder/stateless/vp9: do not request resources for already decoded frames If a frame has its show_existing_frame flag set, then it does not require an output resource. --- src/decoder/stateless/vp9.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index bfd3fb25..dd7d10d4 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -275,11 +275,16 @@ where .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? .num_free_frames(); + let num_required_frames = frames + .iter() + .filter(|f| !f.header.show_existing_frame) + .count(); + if matches!(self.decoding_state, DecodingState::Decoding) - && num_free_frames < frames.len() + && num_free_frames < num_required_frames { return Err(DecodeError::NotEnoughOutputBuffers( - frames.len() - num_free_frames, + num_required_frames - num_free_frames, )); } From 4fec1f58f36be53fdfa6f6baa7f53701fdcf5bdf Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 15:57:28 +0900 Subject: [PATCH 10/22] decoder/stateless/vp9: let the backend decide which resources it needs The decoder code was checking explicitly for output buffers availability before decoding frames, but that requirement is not needed by all backends - for example V4L2 stateless has different requirements. Move this check to the backend by using the new_picture() method to try and allocate all required pictures of a superframe prior to decoding it. --- src/decoder/stateless/vp9.rs | 59 ++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index dd7d10d4..02b33ce9 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -10,7 +10,6 @@ mod vaapi; use std::os::fd::AsFd; use std::os::fd::BorrowedFd; -use anyhow::anyhow; use log::debug; use crate::codec::vp9::parser::BitDepth; @@ -33,10 +32,10 @@ use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; +use super::StatelessBackendError; use super::StatelessDecoderBackendPicture; /// Stateless backend methods specific to VP9. @@ -175,11 +174,10 @@ where } /// Decode a single frame. - fn handle_frame(&mut self, frame: &Frame, timestamp: u64) -> Result<(), DecodeError> { + fn handle_frame(&mut self, frame: &Frame, picture: B::Picture) -> Result<(), DecodeError> { let refresh_frame_flags = frame.header.refresh_frame_flags; Segmentation::update_segmentation(&mut self.codec.segmentation, &frame.header); - let picture = self.backend.new_picture(timestamp)?; let decoded_handle = self.backend.submit_picture( picture, @@ -268,31 +266,40 @@ where // Ask the client to confirm the format before we can process this. DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), DecodingState::Decoding => { - let num_free_frames = self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? - .num_free_frames(); - - let num_required_frames = frames + // First allocate all the pictures we need for this superframe. + let num_required_pictures = frames .iter() .filter(|f| !f.header.show_existing_frame) .count(); - - if matches!(self.decoding_state, DecodingState::Decoding) - && num_free_frames < num_required_frames - { - return Err(DecodeError::NotEnoughOutputBuffers( - num_required_frames - num_free_frames, - )); - } - - for frame in frames { - if frame.header.show_existing_frame { - self.handle_show_existing_frame(frame.header.frame_to_show_map_idx)?; - } else { - self.handle_frame(&frame, timestamp)?; + let frames_with_pictures = frames + .into_iter() + .enumerate() + .map(|(i, frame)| { + if frame.header.show_existing_frame { + Ok((frame, None)) + } else { + self.backend + .new_picture(timestamp) + .map_err(|e| match e { + StatelessBackendError::OutOfResources => { + DecodeError::NotEnoughOutputBuffers( + num_required_pictures - i, + ) + } + e => DecodeError::BackendError(e), + }) + .map(|picture| (frame, Some(picture))) + } + }) + .collect::, _>>()?; + + // Then process each frame. + for (frame, picture) in frames_with_pictures { + match picture { + None => { + self.handle_show_existing_frame(frame.header.frame_to_show_map_idx)? + } + Some(picture) => self.handle_frame(&frame, picture)?, } } } From db441e5c88da01276dea45202ba4e9a3215c0974 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 21:08:20 +0900 Subject: [PATCH 11/22] decoder/stateless/h265: remove pic argument from new_picture method This argument is not needed when creating a new resource. --- src/decoder/stateless/h265.rs | 8 ++------ src/decoder/stateless/h265/dummy.rs | 1 - src/decoder/stateless/h265/vaapi.rs | 6 +----- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index 9cc8dbf9..65dfc5a9 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -113,11 +113,7 @@ pub trait StatelessH265DecoderBackend: fn new_sequence(&mut self, sps: &Sps) -> StatelessBackendResult<()>; /// Called when the decoder determines that a frame or field was found. - fn new_picture( - &mut self, - picture: &PictureData, - timestamp: u64, - ) -> StatelessBackendResult; + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; /// Called by the decoder for every frame or field found. #[allow(clippy::too_many_arguments)] @@ -994,7 +990,7 @@ where self.decode_rps(slice, &pic)?; self.update_dpb_before_decoding(&pic)?; - let mut backend_pic = self.backend.new_picture(&pic, timestamp)?; + let mut backend_pic = self.backend.new_picture(timestamp)?; self.backend.begin_picture( &mut backend_pic, diff --git a/src/decoder/stateless/h265/dummy.rs b/src/decoder/stateless/h265/dummy.rs index 98cf744c..298adde8 100644 --- a/src/decoder/stateless/h265/dummy.rs +++ b/src/decoder/stateless/h265/dummy.rs @@ -27,7 +27,6 @@ impl StatelessH265DecoderBackend for Backend { fn new_picture( &mut self, - _: &crate::codec::h265::picture::PictureData, _: u64, ) -> crate::decoder::stateless::StatelessBackendResult { Ok(()) diff --git a/src/decoder/stateless/h265/vaapi.rs b/src/decoder/stateless/h265/vaapi.rs index 43e5a191..91254f68 100644 --- a/src/decoder/stateless/h265/vaapi.rs +++ b/src/decoder/stateless/h265/vaapi.rs @@ -594,11 +594,7 @@ impl StatelessH265DecoderBackend for Vaapi self.new_sequence(sps, PoolCreationMode::Highest) } - fn new_picture( - &mut self, - _: &PictureData, - timestamp: u64, - ) -> StatelessBackendResult { + fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { let highest_pool = self.highest_pool(); let surface = highest_pool .get_surface() From 75e4e050123be729f05a27e922dee01693ef5a43 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 21:39:36 +0900 Subject: [PATCH 12/22] decoder/stateless/h265: remove unused timestamp parameter from Picture::new_from_slice --- src/codec/h265/picture.rs | 1 - src/decoder/stateless/h265.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/codec/h265/picture.rs b/src/codec/h265/picture.rs index a2a40116..d64618b2 100644 --- a/src/codec/h265/picture.rs +++ b/src/codec/h265/picture.rs @@ -53,7 +53,6 @@ impl PictureData { first_picture_after_eos: bool, prev_tid0_pic: Option<&PictureData>, max_pic_order_cnt_lsb: i32, - _timestamp: u64, ) -> Self { let hdr = &slice.header; let nalu_type = slice.nalu.header.type_; diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index 65dfc5a9..f9c545c7 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -961,7 +961,6 @@ where self.codec.first_picture_after_eos, self.codec.prev_tid_0_pic.as_ref(), self.codec.max_pic_order_cnt_lsb, - timestamp, ); self.codec.first_picture_after_eos = false; From f21da1539498d5af7d512be499ed10590d68fa6b Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 21:32:11 +0900 Subject: [PATCH 13/22] decoder/stateless/h265: remove unneeded PPS fetch This can be set directly from the slice header's information. --- src/decoder/stateless/h265.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index f9c545c7..8f2f6438 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -936,14 +936,7 @@ where return Err(DecodeError::NotEnoughOutputBuffers(1)); } - let pps = self - .codec - .parser - .get_pps(slice.header.pic_parameter_set_id) - .context("Invalid PPS in handle_picture")?; - - let pps_id = pps.pic_parameter_set_id; - self.update_current_set_ids(pps_id)?; + self.update_current_set_ids(slice.header.pic_parameter_set_id)?; self.renegotiate_if_needed(RenegotiationType::CurrentSps)?; // We renegotiated and must return the NALU and wait. From add177c127ec016bb3697604a577fbeafc2891a3 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 21:42:42 +0900 Subject: [PATCH 14/22] decoder/stateless/h265: check if negotiation is needed before resources availability We don't need any resource to perform negotiation, so do that check first. --- src/decoder/stateless/h265.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index 8f2f6438..3b46e14a 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -924,6 +924,13 @@ where timestamp: u64, slice: &Slice, ) -> Result>, DecodeError> { + self.update_current_set_ids(slice.header.pic_parameter_set_id)?; + self.renegotiate_if_needed(RenegotiationType::CurrentSps)?; + // We renegotiated and must return the NALU and wait. + if matches!(self.decoding_state, DecodingState::AwaitingFormat(_)) { + return Err(DecodeError::CheckEvents); + } + let layer = PoolLayer::Layer(self.coded_resolution); if self .backend @@ -936,14 +943,6 @@ where return Err(DecodeError::NotEnoughOutputBuffers(1)); } - self.update_current_set_ids(slice.header.pic_parameter_set_id)?; - self.renegotiate_if_needed(RenegotiationType::CurrentSps)?; - - // We renegotiated and must return the NALU and wait. - if matches!(self.decoding_state, DecodingState::AwaitingFormat(_)) { - return Err(DecodeError::CheckEvents); - } - let pic = PictureData::new_from_slice( slice, self.codec From 02dc0adba0af88d7bce0c2f162167df513a6fe48 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 11 Jul 2024 12:24:03 +0900 Subject: [PATCH 15/22] decoder/stateless/h265: remove cur_pps_id member The only time we access this member is shortly after updating it with update_current_set_ids() from the slice's parameter. So let's just use the slice data instead of adding state that may become invalid. --- src/decoder/stateless/h265.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index 3b46e14a..ba778174 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -286,8 +286,6 @@ pub struct H265DecoderState { /// The current active SPS id. cur_sps_id: u8, - /// The current active PPS id. - cur_pps_id: u8, /// Used to identify first picture in decoding order or first picture that /// follows an EOS NALU. @@ -326,7 +324,6 @@ where rps: Default::default(), dpb: Default::default(), cur_sps_id: Default::default(), - cur_pps_id: Default::default(), first_picture_after_eos: true, first_picture_in_bitstream: true, prev_tid_0_pic: Default::default(), @@ -947,7 +944,7 @@ where slice, self.codec .parser - .get_pps(self.codec.cur_pps_id) + .get_pps(slice.header.pic_parameter_set_id) .context("Invalid PPS")?, self.codec.first_picture_in_bitstream, self.codec.first_picture_after_eos, @@ -992,7 +989,7 @@ where .context("Invalid SPS")?, self.codec .parser - .get_pps(self.codec.cur_pps_id) + .get_pps(slice.header.pic_parameter_set_id) .context("Invalid PPS")?, &self.codec.dpb, &self.codec.rps, @@ -1009,7 +1006,6 @@ where fn update_current_set_ids(&mut self, pps_id: u8) -> anyhow::Result<()> { let pps = self.codec.parser.get_pps(pps_id).context("Invalid PPS")?; - self.codec.cur_pps_id = pps.pic_parameter_set_id; self.codec.cur_sps_id = pps.seq_parameter_set_id; Ok(()) } @@ -1049,7 +1045,7 @@ where .context("Invalid SPS id")?, self.codec .parser - .get_pps(self.codec.cur_pps_id) + .get_pps(slice.header.pic_parameter_set_id) .context("Invalid PPS id")?, &pic.ref_pic_lists.ref_pic_list0, &pic.ref_pic_lists.ref_pic_list1, From 1aeea565bd156c334f775142ba8baf3492ba8d11 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 11 Jul 2024 12:33:44 +0900 Subject: [PATCH 16/22] decoder/stateless/h265: avoid using cur_sps_id where possible This is some frame-local state that we would eventually like to get rid of, so don't use it when we can get the SPS ID through other means. --- src/decoder/stateless/h265.rs | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index ba778174..de06dad7 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -980,17 +980,21 @@ where let mut backend_pic = self.backend.new_picture(timestamp)?; + let pps = self + .codec + .parser + .get_pps(slice.header.pic_parameter_set_id) + .context("invalid PPS ID")?; + let sps = self + .codec + .parser + .get_sps(pps.seq_parameter_set_id) + .context("invalid SPS ID")?; self.backend.begin_picture( &mut backend_pic, &pic, - self.codec - .parser - .get_sps(self.codec.cur_sps_id) - .context("Invalid SPS")?, - self.codec - .parser - .get_pps(slice.header.pic_parameter_set_id) - .context("Invalid PPS")?, + sps, + pps, &self.codec.dpb, &self.codec.rps, slice, @@ -1036,17 +1040,21 @@ where pic.ref_pic_lists = self.build_ref_pic_lists(&slice.header, &pic.pic)?; + let pps = self + .codec + .parser + .get_pps(slice.header.pic_parameter_set_id) + .context("invalid PPS ID")?; + let sps = self + .codec + .parser + .get_sps(pps.seq_parameter_set_id) + .context("invalid SPS ID")?; self.backend.decode_slice( &mut pic.backend_pic, slice, - self.codec - .parser - .get_sps(self.codec.cur_sps_id) - .context("Invalid SPS id")?, - self.codec - .parser - .get_pps(slice.header.pic_parameter_set_id) - .context("Invalid PPS id")?, + sps, + pps, &pic.ref_pic_lists.ref_pic_list0, &pic.ref_pic_lists.ref_pic_list1, )?; From 718ec47da2fc0d7fdc3869beccfb225c7a5521a5 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 11 Jul 2024 12:49:32 +0900 Subject: [PATCH 17/22] decoder/stateless/h265: let the backend decide which resources it needs The decoder code was checking explicitly for output buffers availability before decoding frames, but that requirement is not needed by all backends - for example V4L2 stateless has different requirements. Move this check to the backend by using the new_picture() method to try and allocate a picture before processing it, and return the appropriate error if none is available. --- src/decoder/stateless.rs | 2 ++ src/decoder/stateless/h265.rs | 26 ++++++++++++-------------- src/decoder/stateless/h265/dummy.rs | 2 ++ src/decoder/stateless/h265/vaapi.rs | 17 ++++++++++++++--- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/decoder/stateless.rs b/src/decoder/stateless.rs index 328d3e62..432ac5f0 100644 --- a/src/decoder/stateless.rs +++ b/src/decoder/stateless.rs @@ -45,6 +45,8 @@ use crate::Resolution; pub enum StatelessBackendError { #[error("not enough resources to proceed with the operation now")] OutOfResources, + #[error("no frame pool can satisfy the requested frame resolution {0:?}")] + NoFramePool(Resolution), #[error(transparent)] Other(#[from] anyhow::Error), } diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index de06dad7..572b8708 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -31,6 +31,7 @@ use crate::codec::h265::picture::Reference; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; use crate::decoder::stateless::PoolLayer; +use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; @@ -40,7 +41,6 @@ use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; @@ -113,7 +113,11 @@ pub trait StatelessH265DecoderBackend: fn new_sequence(&mut self, sps: &Sps) -> StatelessBackendResult<()>; /// Called when the decoder determines that a frame or field was found. - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; + fn new_picture( + &mut self, + coded_resolution: Resolution, + timestamp: u64, + ) -> StatelessBackendResult; /// Called by the decoder for every frame or field found. #[allow(clippy::too_many_arguments)] @@ -928,17 +932,13 @@ where return Err(DecodeError::CheckEvents); } - let layer = PoolLayer::Layer(self.coded_resolution); - if self + let mut backend_pic = self .backend - .frame_pool(layer) - .pop() - .ok_or(anyhow!("Pool not found"))? - .num_free_frames() - == 0 - { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } + .new_picture(self.coded_resolution, timestamp) + .map_err(|e| match e { + StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), + e => DecodeError::BackendError(e), + })?; let pic = PictureData::new_from_slice( slice, @@ -978,8 +978,6 @@ where self.decode_rps(slice, &pic)?; self.update_dpb_before_decoding(&pic)?; - let mut backend_pic = self.backend.new_picture(timestamp)?; - let pps = self .codec .parser diff --git a/src/decoder/stateless/h265/dummy.rs b/src/decoder/stateless/h265/dummy.rs index 298adde8..aa5c79f9 100644 --- a/src/decoder/stateless/h265/dummy.rs +++ b/src/decoder/stateless/h265/dummy.rs @@ -16,6 +16,7 @@ use crate::decoder::stateless::StatelessDecoder; use crate::decoder::BlockingMode; use crate::decoder::stateless::h265::StatelessH265DecoderBackend; +use crate::Resolution; impl StatelessH265DecoderBackend for Backend { fn new_sequence( @@ -27,6 +28,7 @@ impl StatelessH265DecoderBackend for Backend { fn new_picture( &mut self, + _: Resolution, _: u64, ) -> crate::decoder::stateless::StatelessBackendResult { Ok(()) diff --git a/src/decoder/stateless/h265/vaapi.rs b/src/decoder/stateless/h265/vaapi.rs index 91254f68..0d51593f 100644 --- a/src/decoder/stateless/h265/vaapi.rs +++ b/src/decoder/stateless/h265/vaapi.rs @@ -38,11 +38,14 @@ use crate::decoder::stateless::h265::RefPicSet; use crate::decoder::stateless::h265::StatelessH265DecoderBackend; use crate::decoder::stateless::h265::H265; use crate::decoder::stateless::NewStatelessDecoderError; +use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; +use crate::decoder::stateless::StatelessDecoderBackend; use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::BlockingMode; +use crate::Resolution; enum ScalingListType { Sps, @@ -594,9 +597,17 @@ impl StatelessH265DecoderBackend for Vaapi self.new_sequence(sps, PoolCreationMode::Highest) } - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { - let highest_pool = self.highest_pool(); - let surface = highest_pool + fn new_picture( + &mut self, + coded_resolution: Resolution, + timestamp: u64, + ) -> StatelessBackendResult { + let layer = PoolLayer::Layer(coded_resolution); + let pool = self + .frame_pool(layer) + .pop() + .ok_or(StatelessBackendError::NoFramePool(coded_resolution))?; + let surface = pool .get_surface() .ok_or(StatelessBackendError::OutOfResources)?; let metadata = self.metadata_state.get_parsed()?; From 895327a69adf8cbfcf79cab3f9c0fc4abdc0396a Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 13:38:38 +0900 Subject: [PATCH 18/22] decoder/stateless/av1: process one OBU per decode call decode() was trying to process a whole temporal unit, which complicates the code quite a bit: the decoder must notably check for available resources for all the frames in the OBU, which requires to parse the input several times. Our contract is that the caller must check how much input has been consumed and re-submit the remainder if it has not been entirely processed. Leverage this and only process one OBU per call to decode(), which simplifies the resource availability check and will also allow us to move it into the backend. --- src/decoder/stateless/av1.rs | 277 +++++++++++++++-------------------- 1 file changed, 119 insertions(+), 158 deletions(-) diff --git a/src/decoder/stateless/av1.rs b/src/decoder/stateless/av1.rs index 5d73475f..a489d804 100644 --- a/src/decoder/stateless/av1.rs +++ b/src/decoder/stateless/av1.rs @@ -7,7 +7,6 @@ use std::os::fd::BorrowedFd; use std::rc::Rc; use anyhow::anyhow; -use anyhow::Context; use crate::codec::av1::parser::FrameHeaderObu; use crate::codec::av1::parser::FrameObu; @@ -16,10 +15,8 @@ use crate::codec::av1::parser::ObuType; use crate::codec::av1::parser::ParsedObu; use crate::codec::av1::parser::Parser; use crate::codec::av1::parser::SequenceHeaderObu; -use crate::codec::av1::parser::NUM_REF_FRAMES; -use crate::Resolution; - use crate::codec::av1::parser::TileGroupObu; +use crate::codec::av1::parser::NUM_REF_FRAMES; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; use crate::decoder::stateless::StatelessBackendResult; @@ -33,6 +30,7 @@ use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::FramePool; use crate::decoder::PoolLayer; +use crate::Resolution; #[cfg(test)] mod dummy; @@ -157,29 +155,6 @@ where B: StatelessAV1DecoderBackend, B::Handle: Clone, { - fn count_frames(&mut self, bitstream: &[u8]) -> usize { - let mut nframes = 0; - let mut consumed = 0; - - while let Ok(obu) = self.codec.parser.parse_obu(&bitstream[consumed..]) { - let obu = match obu { - ParsedObu::Process(obu) => obu, - ParsedObu::Drop(length) => { - consumed += usize::try_from(length).unwrap(); - continue; - } - }; - - if matches!(obu.header.obu_type, ObuType::Frame | ObuType::FrameHeader) { - nframes += 1; - } - - consumed += obu.data.len(); - } - - nframes - } - fn decode_frame_header( &mut self, frame_header: FrameHeaderObu, @@ -327,10 +302,19 @@ where type Handle = B::Handle; type FramePool = B::FramePool; + /// Decode an AV1 stream. + /// + /// `bitstream` should initially be submitted as a whole temporal unit, however a call to this + /// method will only consume a single OBU. The caller must be careful to check the return value + /// and resubmit the remainder if the whole bitstream has not been consumed. fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { - let mut consumed = 0; + let obu = match self.codec.parser.parse_obu(bitstream)? { + ParsedObu::Process(obu) => obu, + // This OBU should be dropped. + ParsedObu::Drop(length) => return Ok(length as usize), + }; + let obu_length = obu.data.len(); - let nframes = self.count_frames(bitstream); /* we do not know the resolution at this point, as we haven't parsed the * frames yet. Be conservative and check whether we have enough frames * across all layers */ @@ -340,160 +324,137 @@ where .iter() .map(|x| x.num_free_frames()) .min() - .ok_or(anyhow!("No pool found"))?; + .ok_or(anyhow!("no pool found"))?; - if matches!(self.decoding_state, DecodingState::Decoding) && num_free_frames < nframes { - return Err(DecodeError::NotEnoughOutputBuffers( - nframes - num_free_frames, - )); + if matches!(self.decoding_state, DecodingState::Decoding) && num_free_frames < 1 { + return Err(DecodeError::NotEnoughOutputBuffers(1)); } - while let Ok(obu) = self.codec.parser.parse_obu(&bitstream[consumed..]) { - let obu = match obu { - ParsedObu::Process(obu) => obu, - // This OBU should be dropped. - ParsedObu::Drop(length) => { - consumed += usize::try_from(length).context("OBU length too large")?; - continue; - } - }; - - let obu_length = obu.data.len(); - - let is_decode_op = matches!( - obu.header.obu_type, - ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup - ); + let is_decode_op = matches!( + obu.header.obu_type, + ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup + ); - if is_decode_op { - match self.decoding_state { - /* we want to be here */ - DecodingState::Decoding => (), + if is_decode_op { + match self.decoding_state { + /* we want to be here */ + DecodingState::Decoding => (), - /* otherwise... */ - DecodingState::AwaitingStreamInfo => { - /* Skip input until we get information from the stream. */ - consumed += obu_length; - continue; - } - /* Ask the client to confirm the format before we can process this. */ - DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), - DecodingState::Reset => { - let mut parser = self.codec.parser.clone(); - - let is_key_frame = match obu.header.obu_type { - ObuType::Frame => { - let frame = parser.parse_frame_obu(obu.clone())?; - frame.header.frame_type == FrameType::KeyFrame - } - ObuType::FrameHeader => { - let fh = parser.parse_frame_header_obu(&obu)?; - fh.frame_type == FrameType::KeyFrame - } - _ => false, - }; - - /* we can only resume from key frames */ - if !is_key_frame { - consumed += obu_length; - continue; - } else { - self.decoding_state = DecodingState::Decoding; - } - } + /* otherwise... */ + DecodingState::AwaitingStreamInfo => { + /* Skip input until we get information from the stream. */ + return Ok(obu_length); } - } - - match obu.header.obu_type { - ObuType::SequenceHeader => { - let sequence = self.codec.parser.parse_sequence_header_obu(&obu)?; - let sequence_differs = match &self.codec.sequence { - Some(old_sequence) => **old_sequence != *sequence, - None => true, - }; - - if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) - || sequence_differs - { - if self.codec.current_pic.is_some() { - return Err(DecodeError::DecoderError(anyhow!( - "Broken stream: a picture is being decoded while a new sequence header is encountered" - ))); + /* Ask the client to confirm the format before we can process this. */ + DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), + DecodingState::Reset => { + let mut parser = self.codec.parser.clone(); + + let is_key_frame = match obu.header.obu_type { + ObuType::Frame => { + let frame = parser.parse_frame_obu(obu.clone())?; + frame.header.frame_type == FrameType::KeyFrame } - - /* make sure we sync *before* we clear any state in the backend */ - for f in &mut self.ready_queue.queue { - /* TODO: this fixes av1-1-b8-03-sizeup on Intel - * gen12, but we apparently do not do the same in - * VP9. How is it that we do not get similar crashes there? - * - * TODO: syncing before calling new_sequence() in VP9 may fix some tests - */ - f.sync()?; + ObuType::FrameHeader => { + let fh = parser.parse_frame_header_obu(&obu)?; + fh.frame_type == FrameType::KeyFrame } + _ => false, + }; - log::debug!( - "Found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}", - Resolution::from(( - sequence.max_frame_width_minus_1 as u32 + 1, - sequence.max_frame_height_minus_1 as u32 + 1 - )), - sequence.seq_profile, - sequence.bit_depth - ); - /* there is nothing to drain, much like vp8 and vp9 */ - self.codec.highest_spatial_layer = - self.codec.parser.highest_operating_point(); - self.backend - .new_sequence(&sequence, self.codec.highest_spatial_layer)?; - self.await_format_change(sequence); + /* we can only resume from key frames */ + if !is_key_frame { + return Ok(obu_length); + } else { + self.decoding_state = DecodingState::Decoding; } } - ObuType::TemporalDelimiter => { - self.codec.parser.parse_temporal_delimiter_obu(&obu)? - } - ObuType::FrameHeader => { + } + } + + match obu.header.obu_type { + ObuType::SequenceHeader => { + let sequence = self.codec.parser.parse_sequence_header_obu(&obu)?; + let sequence_differs = match &self.codec.sequence { + Some(old_sequence) => **old_sequence != *sequence, + None => true, + }; + + if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) + || sequence_differs + { if self.codec.current_pic.is_some() { - /* submit this frame immediately, as we need to update the - * DPB and the reference info state *before* processing the - * next frame */ - self.submit_frame(timestamp)?; + return Err(DecodeError::DecoderError(anyhow!( + "broken stream: a picture is being decoded while a new sequence header is encountered" + ))); } - let frame_header = self.codec.parser.parse_frame_header_obu(&obu)?; - self.decode_frame_header(frame_header, timestamp)?; - } - ObuType::TileGroup => { - let tile_group = self.codec.parser.parse_tile_group_obu(obu)?; - self.decode_tile_group(tile_group)?; + + /* make sure we sync *before* we clear any state in the backend */ + for f in &mut self.ready_queue.queue { + /* TODO: this fixes av1-1-b8-03-sizeup on Intel + * gen12, but we apparently do not do the same in + * VP9. How is it that we do not get similar crashes there? + * + * TODO: syncing before calling new_sequence() in VP9 may fix some tests + */ + f.sync()?; + } + + log::debug!( + "found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}", + Resolution::from(( + sequence.max_frame_width_minus_1 as u32 + 1, + sequence.max_frame_height_minus_1 as u32 + 1 + )), + sequence.seq_profile, + sequence.bit_depth + ); + /* there is nothing to drain, much like vp8 and vp9 */ + self.codec.highest_spatial_layer = self.codec.parser.highest_operating_point(); + self.backend + .new_sequence(&sequence, self.codec.highest_spatial_layer)?; + self.await_format_change(sequence); } - ObuType::Frame => { - let frame = self.codec.parser.parse_frame_obu(obu)?; - self.decode_frame(frame, timestamp)?; + } + ObuType::TemporalDelimiter => self.codec.parser.parse_temporal_delimiter_obu(&obu)?, + ObuType::FrameHeader => { + if self.codec.current_pic.is_some() { /* submit this frame immediately, as we need to update the * DPB and the reference info state *before* processing the * next frame */ self.submit_frame(timestamp)?; } - ObuType::TileList => { - return Err(DecodeError::DecoderError(anyhow!( - "Large tile scale mode is not supported" - ))); - } - other => { - log::debug!("Skipping OBU of type {:?}", other); - } + let frame_header = self.codec.parser.parse_frame_header_obu(&obu)?; + self.decode_frame_header(frame_header, timestamp)?; + } + ObuType::TileGroup => { + let tile_group = self.codec.parser.parse_tile_group_obu(obu)?; + self.decode_tile_group(tile_group)?; + } + ObuType::Frame => { + let frame = self.codec.parser.parse_frame_obu(obu)?; + self.decode_frame(frame, timestamp)?; + /* submit this frame immediately, as we need to update the + * DPB and the reference info state *before* processing the + * next frame */ + self.submit_frame(timestamp)?; + } + ObuType::TileList => { + return Err(DecodeError::DecoderError(anyhow!( + "large tile scale mode is not supported" + ))); + } + other => { + log::debug!("skipping OBU of type {:?}", other); } - - consumed += obu_length; } - /* we may already have dispatched work if we got ObuType::Frame */ - if self.codec.current_pic.is_some() { - /* dispatch work to the backend */ + /* Submit the last frame if we have reached the end of the temporal unit. */ + if bitstream.len() == obu_length && self.codec.current_pic.is_some() { self.submit_frame(timestamp)?; } - Ok(consumed) + Ok(obu_length) } fn flush(&mut self) -> Result<(), super::DecodeError> { From cebb5df1b926747c60f66b8b001a4e4e7bd66a52 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 12:45:06 +0900 Subject: [PATCH 19/22] decoder/stateless/av1/vaapi: optimize new_picture a bit --- src/decoder/stateless/av1/vaapi.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/decoder/stateless/av1/vaapi.rs b/src/decoder/stateless/av1/vaapi.rs index aed31961..ec340b39 100644 --- a/src/decoder/stateless/av1/vaapi.rs +++ b/src/decoder/stateless/av1/vaapi.rs @@ -532,30 +532,25 @@ impl StatelessAV1DecoderBackend for VaapiB reference_frames: &[Option; NUM_REF_FRAMES], highest_spatial_layer: Option, ) -> crate::decoder::stateless::StatelessBackendResult { - let surface = match highest_spatial_layer { + let pool = match highest_spatial_layer { Some(_) => { let layer = Resolution { width: hdr.upscaled_width, height: hdr.frame_height, }; - let pool = self - .pool(layer) + self.pool(layer) .ok_or(StatelessBackendError::Other(anyhow!( "No pool available for this layer" - )))?; - - pool.get_surface() - .ok_or(StatelessBackendError::OutOfResources)? - } - None => { - let highest_pool = self.highest_pool(); - highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)? + )))? } + None => self.highest_pool(), }; + let surface = pool + .get_surface() + .ok_or(StatelessBackendError::OutOfResources)?; + let metadata = self.metadata_state.get_parsed()?; let mut picture = VaPicture::new(timestamp, Rc::clone(&metadata.context), surface); From 25d67b87a1fe64c305b67ec1ca6697a7142e60c1 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 10 Jul 2024 12:56:15 +0900 Subject: [PATCH 20/22] decoder/stateless/av1: split backend picture creation from initialization We want to create pictures early so the backend can check whether all required resources are available before processing. This requires splitting new_picture into a method that allocates the picture and its resources, and another one that initializes the picture parameters, so they can be called from two different sites. --- src/decoder/stateless/av1.rs | 27 ++++++++++++++++++------- src/decoder/stateless/av1/dummy.rs | 12 +++++++++-- src/decoder/stateless/av1/vaapi.rs | 32 +++++++++++++++++++----------- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/decoder/stateless/av1.rs b/src/decoder/stateless/av1.rs index a489d804..37debd80 100644 --- a/src/decoder/stateless/av1.rs +++ b/src/decoder/stateless/av1.rs @@ -51,16 +51,24 @@ pub trait StatelessAV1DecoderBackend: highest_spatial_layer: Option, ) -> StatelessBackendResult<()>; - /// Called when the decoder determines that a new picture was found. + /// Called when the decoder determines that a new picture was found. The backend allocates all + /// the resources it needs to process that picture. fn new_picture( &mut self, - sequence: &SequenceHeaderObu, - picture: &FrameHeaderObu, + hdr: &FrameHeaderObu, timestamp: u64, - reference_frames: &[Option; NUM_REF_FRAMES], highest_spatial_layer: Option, ) -> StatelessBackendResult; + /// Called to set the global parameters of a picture. + fn begin_picture( + &mut self, + picture: &mut Self::Picture, + sequence: &SequenceHeaderObu, + hdr: &FrameHeaderObu, + reference_frames: &[Option; NUM_REF_FRAMES], + ) -> StatelessBackendResult<()>; + /// Called to dispatch a decode operation to the backend. #[allow(clippy::too_many_arguments)] fn decode_tile_group( @@ -176,14 +184,19 @@ where handle: ref_frame.clone(), }); } else if let Some(sequence) = &self.codec.sequence { - let backend_picture = self.backend.new_picture( - sequence, + let mut backend_picture = self.backend.new_picture( &frame_header, timestamp, - &self.codec.reference_frames, self.codec.highest_spatial_layer, )?; + self.backend.begin_picture( + &mut backend_picture, + sequence, + &frame_header, + &self.codec.reference_frames, + )?; + self.codec.current_pic = Some(CurrentPicState::RegularFrame { header: frame_header.clone(), backend_picture, diff --git a/src/decoder/stateless/av1/dummy.rs b/src/decoder/stateless/av1/dummy.rs index d87dfe41..9f0b1254 100644 --- a/src/decoder/stateless/av1/dummy.rs +++ b/src/decoder/stateless/av1/dummy.rs @@ -28,15 +28,23 @@ impl StatelessAV1DecoderBackend for Backend { fn new_picture( &mut self, - _: &crate::codec::av1::parser::SequenceHeaderObu, _: &crate::codec::av1::parser::FrameHeaderObu, _: u64, - _: &[Option; crate::codec::av1::parser::NUM_REF_FRAMES], _: Option, ) -> crate::decoder::stateless::StatelessBackendResult { Ok(()) } + fn begin_picture( + &mut self, + _: &mut Self::Picture, + _: &crate::codec::av1::parser::SequenceHeaderObu, + _: &crate::codec::av1::parser::FrameHeaderObu, + _: &[Option; crate::codec::av1::parser::NUM_REF_FRAMES], + ) -> crate::decoder::stateless::StatelessBackendResult<()> { + Ok(()) + } + fn decode_tile_group( &mut self, _: &mut Self::Picture, diff --git a/src/decoder/stateless/av1/vaapi.rs b/src/decoder/stateless/av1/vaapi.rs index ec340b39..ffc6cad9 100644 --- a/src/decoder/stateless/av1/vaapi.rs +++ b/src/decoder/stateless/av1/vaapi.rs @@ -31,6 +31,7 @@ use crate::decoder::stateless::av1::Av1; use crate::decoder::stateless::av1::StatelessAV1DecoderBackend; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendError; +use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::BlockingMode; @@ -507,7 +508,7 @@ impl StatelessAV1DecoderBackend for VaapiB &mut self, sequence: &Rc, highest_spatial_layer: Option, - ) -> crate::decoder::stateless::StatelessBackendResult<()> { + ) -> StatelessBackendResult<()> { let pool_creation_mode = match highest_spatial_layer { Some(highest_layer) => { /* The spec mandates a 2:1 or 1.5:1 ratio, let's go with 2:1 to @@ -526,12 +527,10 @@ impl StatelessAV1DecoderBackend for VaapiB fn new_picture( &mut self, - sequence: &SequenceHeaderObu, hdr: &FrameHeaderObu, timestamp: u64, - reference_frames: &[Option; NUM_REF_FRAMES], highest_spatial_layer: Option, - ) -> crate::decoder::stateless::StatelessBackendResult { + ) -> StatelessBackendResult { let pool = match highest_spatial_layer { Some(_) => { let layer = Resolution { @@ -552,11 +551,23 @@ impl StatelessAV1DecoderBackend for VaapiB .ok_or(StatelessBackendError::OutOfResources)?; let metadata = self.metadata_state.get_parsed()?; - let mut picture = VaPicture::new(timestamp, Rc::clone(&metadata.context), surface); + Ok(VaPicture::new( + timestamp, + Rc::clone(&metadata.context), + surface, + )) + } - let surface_id = picture.surface().id(); + fn begin_picture( + &mut self, + picture: &mut Self::Picture, + sequence: &SequenceHeaderObu, + hdr: &FrameHeaderObu, + reference_frames: &[Option; NUM_REF_FRAMES], + ) -> StatelessBackendResult<()> { + let metadata = self.metadata_state.get_parsed()?; - let pic_param = build_pic_param(hdr, sequence, surface_id, reference_frames) + let pic_param = build_pic_param(hdr, sequence, picture.surface().id(), reference_frames) .context("Failed to build picture parameter")?; let pic_param = metadata .context @@ -564,7 +575,7 @@ impl StatelessAV1DecoderBackend for VaapiB .context("Failed to create picture parameter buffer")?; picture.add_buffer(pic_param); - Ok(picture) + Ok(()) } fn decode_tile_group( @@ -593,10 +604,7 @@ impl StatelessAV1DecoderBackend for VaapiB Ok(()) } - fn submit_picture( - &mut self, - picture: Self::Picture, - ) -> crate::decoder::stateless::StatelessBackendResult { + fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult { self.process_picture::(picture) } } From 009fb6726dd989901029bb796c51f23b27c8aac2 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 11 Jul 2024 12:49:32 +0900 Subject: [PATCH 21/22] decoder/stateless/av1: let the backend decide which resources it needs The decoder code was checking explicitly for output buffers availability before decoding frames, but that requirement is not needed by all backends - for example V4L2 stateless has different requirements. Move this check to the backend by using the new_picture() method to try and allocate a picture before processing it, and return the appropriate error if none is available. --- src/decoder/stateless/av1.rs | 37 +++++++++++------------------- src/decoder/stateless/av1/vaapi.rs | 4 +--- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/decoder/stateless/av1.rs b/src/decoder/stateless/av1.rs index 37debd80..3b15fb43 100644 --- a/src/decoder/stateless/av1.rs +++ b/src/decoder/stateless/av1.rs @@ -19,6 +19,7 @@ use crate::codec::av1::parser::TileGroupObu; use crate::codec::av1::parser::NUM_REF_FRAMES; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; @@ -28,7 +29,6 @@ use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; -use crate::decoder::FramePool; use crate::decoder::PoolLayer; use crate::Resolution; @@ -167,7 +167,7 @@ where &mut self, frame_header: FrameHeaderObu, timestamp: u64, - ) -> anyhow::Result<()> { + ) -> Result<(), DecodeError> { log::debug!( "Processing frame {} with timestamp {}", self.codec.frame_count, @@ -184,11 +184,13 @@ where handle: ref_frame.clone(), }); } else if let Some(sequence) = &self.codec.sequence { - let mut backend_picture = self.backend.new_picture( - &frame_header, - timestamp, - self.codec.highest_spatial_layer, - )?; + let mut backend_picture = self + .backend + .new_picture(&frame_header, timestamp, self.codec.highest_spatial_layer) + .map_err(|e| match e { + StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), + e => DecodeError::BackendError(e), + })?; self.backend.begin_picture( &mut backend_picture, @@ -227,7 +229,7 @@ where Ok(()) } - fn decode_frame(&mut self, frame: FrameObu, timestamp: u64) -> anyhow::Result<()> { + fn decode_frame(&mut self, frame: FrameObu, timestamp: u64) -> Result<(), DecodeError> { let FrameObu { header, tile_group } = frame; self.decode_frame_header(header, timestamp)?; self.decode_tile_group(tile_group)?; @@ -320,7 +322,7 @@ where /// `bitstream` should initially be submitted as a whole temporal unit, however a call to this /// method will only consume a single OBU. The caller must be careful to check the return value /// and resubmit the remainder if the whole bitstream has not been consumed. - fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { + fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { let obu = match self.codec.parser.parse_obu(bitstream)? { ParsedObu::Process(obu) => obu, // This OBU should be dropped. @@ -328,21 +330,6 @@ where }; let obu_length = obu.data.len(); - /* we do not know the resolution at this point, as we haven't parsed the - * frames yet. Be conservative and check whether we have enough frames - * across all layers */ - let num_free_frames = self - .backend - .frame_pool(PoolLayer::All) - .iter() - .map(|x| x.num_free_frames()) - .min() - .ok_or(anyhow!("no pool found"))?; - - if matches!(self.decoding_state, DecodingState::Decoding) && num_free_frames < 1 { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } - let is_decode_op = matches!( obu.header.obu_type, ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup @@ -385,6 +372,8 @@ where } } + /* We are in `Decoding` state if we reached here */ + match obu.header.obu_type { ObuType::SequenceHeader => { let sequence = self.codec.parser.parse_sequence_header_obu(&obu)?; diff --git a/src/decoder/stateless/av1/vaapi.rs b/src/decoder/stateless/av1/vaapi.rs index ffc6cad9..691ba669 100644 --- a/src/decoder/stateless/av1/vaapi.rs +++ b/src/decoder/stateless/av1/vaapi.rs @@ -539,9 +539,7 @@ impl StatelessAV1DecoderBackend for VaapiB }; self.pool(layer) - .ok_or(StatelessBackendError::Other(anyhow!( - "No pool available for this layer" - )))? + .ok_or(StatelessBackendError::NoFramePool(layer))? } None => self.highest_pool(), }; From 353dc6d2305c6827f530355a20d5cda3e97b0e35 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Tue, 16 Jul 2024 09:56:52 +0900 Subject: [PATCH 22/22] decoder/stateless: add error type for new_picture methods Failures in new_picture are usually due to temporary unavailability of required resources and are recoverable. Introduce a new error type to better handle this, and try to be more precise about the type of resource missing so the client can handle each case properly. --- src/decoder/stateless.rs | 38 ++++++++++++++++++++++++++--- src/decoder/stateless/av1.rs | 16 ++++++------ src/decoder/stateless/av1/dummy.rs | 2 +- src/decoder/stateless/av1/vaapi.rs | 9 ++++--- src/decoder/stateless/h264.rs | 15 ++++-------- src/decoder/stateless/h264/dummy.rs | 5 ++-- src/decoder/stateless/h264/vaapi.rs | 9 ++++--- src/decoder/stateless/h265.rs | 15 +++--------- src/decoder/stateless/h265/dummy.rs | 2 +- src/decoder/stateless/h265/vaapi.rs | 9 ++++--- src/decoder/stateless/vp8.rs | 10 +++----- src/decoder/stateless/vp8/dummy.rs | 3 ++- src/decoder/stateless/vp8/vaapi.rs | 7 +++--- src/decoder/stateless/vp9.rs | 12 ++++----- src/decoder/stateless/vp9/dummy.rs | 3 ++- src/decoder/stateless/vp9/vaapi.rs | 7 +++--- 16 files changed, 91 insertions(+), 71 deletions(-) diff --git a/src/decoder/stateless.rs b/src/decoder/stateless.rs index 432ac5f0..79b7ab16 100644 --- a/src/decoder/stateless.rs +++ b/src/decoder/stateless.rs @@ -40,13 +40,28 @@ use crate::decoder::StreamInfo; use crate::DecodedFormat; use crate::Resolution; -/// Error returned by stateless backend methods. +/// Error returned by `new_picture` methods of the backend, usually to indicate which kind of +/// resource is needed before the picture can be successfully created. #[derive(Error, Debug)] -pub enum StatelessBackendError { - #[error("not enough resources to proceed with the operation now")] - OutOfResources, +pub enum NewPictureError { + /// Indicates that the backend needs one output buffer to be returned to the pool before it can + /// proceed. + #[error("need one output buffer to be returned before operation can proceed")] + OutOfOutputBuffers, + /// No frame pool could satisfy the frame requirements. This indicate either an unhandled DRC + /// or an invalid stream. #[error("no frame pool can satisfy the requested frame resolution {0:?}")] NoFramePool(Resolution), + /// An unrecoverable backend error has occured. + #[error(transparent)] + BackendError(#[from] anyhow::Error), +} + +pub type NewPictureResult = Result; + +/// Error returned by stateless backend methods. +#[derive(Error, Debug)] +pub enum StatelessBackendError { #[error(transparent)] Other(#[from] anyhow::Error), } @@ -87,6 +102,21 @@ pub enum DecodeError { BackendError(#[from] StatelessBackendError), } +/// Convenience conversion for codecs that process a single frame per decode call. +impl From for DecodeError { + fn from(err: NewPictureError) -> Self { + match err { + NewPictureError::OutOfOutputBuffers => DecodeError::NotEnoughOutputBuffers(1), + e @ NewPictureError::NoFramePool(_) => { + DecodeError::BackendError(StatelessBackendError::Other(anyhow::anyhow!(e))) + } + NewPictureError::BackendError(e) => { + DecodeError::BackendError(StatelessBackendError::Other(e)) + } + } + } +} + mod private { use super::*; diff --git a/src/decoder/stateless/av1.rs b/src/decoder/stateless/av1.rs index 3b15fb43..d8f779c8 100644 --- a/src/decoder/stateless/av1.rs +++ b/src/decoder/stateless/av1.rs @@ -19,7 +19,7 @@ use crate::codec::av1::parser::TileGroupObu; use crate::codec::av1::parser::NUM_REF_FRAMES; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; -use crate::decoder::stateless::StatelessBackendError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; @@ -58,7 +58,7 @@ pub trait StatelessAV1DecoderBackend: hdr: &FrameHeaderObu, timestamp: u64, highest_spatial_layer: Option, - ) -> StatelessBackendResult; + ) -> NewPictureResult; /// Called to set the global parameters of a picture. fn begin_picture( @@ -184,13 +184,11 @@ where handle: ref_frame.clone(), }); } else if let Some(sequence) = &self.codec.sequence { - let mut backend_picture = self - .backend - .new_picture(&frame_header, timestamp, self.codec.highest_spatial_layer) - .map_err(|e| match e { - StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), - e => DecodeError::BackendError(e), - })?; + let mut backend_picture = self.backend.new_picture( + &frame_header, + timestamp, + self.codec.highest_spatial_layer, + )?; self.backend.begin_picture( &mut backend_picture, diff --git a/src/decoder/stateless/av1/dummy.rs b/src/decoder/stateless/av1/dummy.rs index 9f0b1254..9938ed7a 100644 --- a/src/decoder/stateless/av1/dummy.rs +++ b/src/decoder/stateless/av1/dummy.rs @@ -31,7 +31,7 @@ impl StatelessAV1DecoderBackend for Backend { _: &crate::codec::av1::parser::FrameHeaderObu, _: u64, _: Option, - ) -> crate::decoder::stateless::StatelessBackendResult { + ) -> crate::decoder::stateless::NewPictureResult { Ok(()) } diff --git a/src/decoder/stateless/av1/vaapi.rs b/src/decoder/stateless/av1/vaapi.rs index 691ba669..80be00e6 100644 --- a/src/decoder/stateless/av1/vaapi.rs +++ b/src/decoder/stateless/av1/vaapi.rs @@ -29,8 +29,9 @@ use crate::codec::av1::parser::NUM_REF_FRAMES; use crate::codec::av1::parser::SEG_LVL_MAX; use crate::decoder::stateless::av1::Av1; use crate::decoder::stateless::av1::StatelessAV1DecoderBackend; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -530,7 +531,7 @@ impl StatelessAV1DecoderBackend for VaapiB hdr: &FrameHeaderObu, timestamp: u64, highest_spatial_layer: Option, - ) -> StatelessBackendResult { + ) -> NewPictureResult { let pool = match highest_spatial_layer { Some(_) => { let layer = Resolution { @@ -539,14 +540,14 @@ impl StatelessAV1DecoderBackend for VaapiB }; self.pool(layer) - .ok_or(StatelessBackendError::NoFramePool(layer))? + .ok_or(NewPictureError::NoFramePool(layer))? } None => self.highest_pool(), }; let surface = pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; Ok(VaPicture::new( diff --git a/src/decoder/stateless/h264.rs b/src/decoder/stateless/h264.rs index d5646e34..9fc88dc6 100644 --- a/src/decoder/stateless/h264.rs +++ b/src/decoder/stateless/h264.rs @@ -40,12 +40,13 @@ use crate::codec::h264::picture::RcPictureData; use crate::codec::h264::picture::Reference; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; @@ -54,8 +55,6 @@ use crate::decoder::DecoderEvent; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - fn get_raster_from_zigzag_8x8(src: [u8; 64], dst: &mut [u8; 64]) { const ZIGZAG_8X8: [usize; 64] = [ 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, @@ -84,7 +83,7 @@ pub trait StatelessH264DecoderBackend: fn new_sequence(&mut self, sps: &Rc) -> StatelessBackendResult<()>; /// Called when the decoder determines that a frame or field was found. - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult; /// Called when the decoder determines that a second field was found. /// Indicates that the underlying BackendHandle is to be shared between the @@ -94,7 +93,7 @@ pub trait StatelessH264DecoderBackend: &mut self, timestamp: u64, first_field: &Self::Handle, - ) -> StatelessBackendResult; + ) -> NewPictureResult; /// Called by the decoder when starting a new frame or field. fn start_picture( @@ -1099,11 +1098,7 @@ where self.backend.new_field_picture(timestamp, &first_field.1) } else { self.backend.new_picture(timestamp) - } - .map_err(|e| match e { - StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), - e => DecodeError::BackendError(e), - })?; + }?; let nalu_hdr = &slice.nalu.header; diff --git a/src/decoder/stateless/h264/dummy.rs b/src/decoder/stateless/h264/dummy.rs index 069b18bf..b6e1d39c 100644 --- a/src/decoder/stateless/h264/dummy.rs +++ b/src/decoder/stateless/h264/dummy.rs @@ -19,6 +19,7 @@ use crate::codec::h264::parser::Sps; use crate::codec::h264::picture::PictureData; use crate::decoder::stateless::h264::StatelessH264DecoderBackend; use crate::decoder::stateless::h264::H264; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; @@ -41,7 +42,7 @@ impl StatelessH264DecoderBackend for Backend { Ok(()) } - fn new_field_picture(&mut self, _: u64, _: &Self::Handle) -> StatelessBackendResult<()> { + fn new_field_picture(&mut self, _: u64, _: &Self::Handle) -> NewPictureResult<()> { Ok(()) } @@ -63,7 +64,7 @@ impl StatelessH264DecoderBackend for Backend { }) } - fn new_picture(&mut self, _: u64) -> StatelessBackendResult<()> { + fn new_picture(&mut self, _: u64) -> NewPictureResult<()> { Ok(()) } } diff --git a/src/decoder/stateless/h264/vaapi.rs b/src/decoder/stateless/h264/vaapi.rs index a8435445..f909f501 100644 --- a/src/decoder/stateless/h264/vaapi.rs +++ b/src/decoder/stateless/h264/vaapi.rs @@ -35,8 +35,9 @@ use crate::codec::h264::picture::PictureData; use crate::codec::h264::picture::Reference; use crate::decoder::stateless::h264::StatelessH264DecoderBackend; use crate::decoder::stateless::h264::H264; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -540,11 +541,11 @@ impl StatelessH264DecoderBackend for Vaapi self.process_picture::(picture) } - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult { let highest_pool = self.highest_pool(); let surface = highest_pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; @@ -559,7 +560,7 @@ impl StatelessH264DecoderBackend for Vaapi &mut self, timestamp: u64, first_field: &Self::Handle, - ) -> StatelessBackendResult { + ) -> NewPictureResult { // Decode to the same surface as the first field picture. Ok(first_field .borrow() diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index 572b8708..d7398777 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -30,12 +30,13 @@ use crate::codec::h265::picture::PictureData; use crate::codec::h265::picture::Reference; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; @@ -44,8 +45,6 @@ use crate::decoder::DecoderEvent; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - const MAX_DPB_SIZE: usize = 16; // Equation 5-8 @@ -117,7 +116,7 @@ pub trait StatelessH265DecoderBackend: &mut self, coded_resolution: Resolution, timestamp: u64, - ) -> StatelessBackendResult; + ) -> NewPictureResult; /// Called by the decoder for every frame or field found. #[allow(clippy::too_many_arguments)] @@ -932,13 +931,7 @@ where return Err(DecodeError::CheckEvents); } - let mut backend_pic = self - .backend - .new_picture(self.coded_resolution, timestamp) - .map_err(|e| match e { - StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), - e => DecodeError::BackendError(e), - })?; + let mut backend_pic = self.backend.new_picture(self.coded_resolution, timestamp)?; let pic = PictureData::new_from_slice( slice, diff --git a/src/decoder/stateless/h265/dummy.rs b/src/decoder/stateless/h265/dummy.rs index aa5c79f9..c0bd12e6 100644 --- a/src/decoder/stateless/h265/dummy.rs +++ b/src/decoder/stateless/h265/dummy.rs @@ -30,7 +30,7 @@ impl StatelessH265DecoderBackend for Backend { &mut self, _: Resolution, _: u64, - ) -> crate::decoder::stateless::StatelessBackendResult { + ) -> crate::decoder::stateless::NewPictureResult { Ok(()) } diff --git a/src/decoder/stateless/h265/vaapi.rs b/src/decoder/stateless/h265/vaapi.rs index 0d51593f..d2c3f92c 100644 --- a/src/decoder/stateless/h265/vaapi.rs +++ b/src/decoder/stateless/h265/vaapi.rs @@ -37,9 +37,10 @@ use crate::decoder::stateless::h265::RefPicListEntry; use crate::decoder::stateless::h265::RefPicSet; use crate::decoder::stateless::h265::StatelessH265DecoderBackend; use crate::decoder::stateless::h265::H265; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::PoolLayer; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; @@ -601,15 +602,15 @@ impl StatelessH265DecoderBackend for Vaapi &mut self, coded_resolution: Resolution, timestamp: u64, - ) -> StatelessBackendResult { + ) -> NewPictureResult { let layer = PoolLayer::Layer(coded_resolution); let pool = self .frame_pool(layer) .pop() - .ok_or(StatelessBackendError::NoFramePool(coded_resolution))?; + .ok_or(NewPictureError::NoFramePool(coded_resolution))?; let surface = pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; Ok(VaapiH265Picture { diff --git a/src/decoder/stateless/vp8.rs b/src/decoder/stateless/vp8.rs index 078a5038..d3b5caf6 100644 --- a/src/decoder/stateless/vp8.rs +++ b/src/decoder/stateless/vp8.rs @@ -17,8 +17,8 @@ use crate::codec::vp8::parser::Parser; use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; @@ -40,7 +40,7 @@ pub trait StatelessVp8DecoderBackend: fn new_sequence(&mut self, header: &Header) -> StatelessBackendResult<()>; /// Called when the decoder determines that a frame or field was found. - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult; /// Called when the decoder wants the backend to finish the decoding /// operations for `picture`. @@ -180,11 +180,7 @@ where fn handle_frame(&mut self, frame: Frame, timestamp: u64) -> Result<(), DecodeError> { let show_frame = frame.header.show_frame; - let picture = self.backend.new_picture(timestamp).map_err(|e| match e { - StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1), - e => DecodeError::BackendError(e), - })?; - + let picture = self.backend.new_picture(timestamp)?; let decoded_handle = self.backend.submit_picture( picture, &frame.header, diff --git a/src/decoder/stateless/vp8/dummy.rs b/src/decoder/stateless/vp8/dummy.rs index 1892b279..c4a1f4f0 100644 --- a/src/decoder/stateless/vp8/dummy.rs +++ b/src/decoder/stateless/vp8/dummy.rs @@ -15,6 +15,7 @@ use crate::codec::vp8::parser::MbLfAdjustments; use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::vp8::StatelessVp8DecoderBackend; use crate::decoder::stateless::vp8::Vp8; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; @@ -25,7 +26,7 @@ impl StatelessVp8DecoderBackend for Backend { Ok(()) } - fn new_picture(&mut self, _: u64) -> StatelessBackendResult { + fn new_picture(&mut self, _: u64) -> NewPictureResult { Ok(()) } diff --git a/src/decoder/stateless/vp8/vaapi.rs b/src/decoder/stateless/vp8/vaapi.rs index 8922b8d0..a6187c4e 100644 --- a/src/decoder/stateless/vp8/vaapi.rs +++ b/src/decoder/stateless/vp8/vaapi.rs @@ -24,8 +24,9 @@ use crate::codec::vp8::parser::MbLfAdjustments; use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::vp8::StatelessVp8DecoderBackend; use crate::decoder::stateless::vp8::Vp8; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -217,11 +218,11 @@ impl StatelessVp8DecoderBackend for VaapiB self.new_sequence(header, PoolCreationMode::Highest) } - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult { let highest_pool = self.highest_pool(); let surface = highest_pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index 02b33ce9..89c1bf37 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -22,11 +22,14 @@ use crate::codec::vp9::parser::MAX_SEGMENTS; use crate::codec::vp9::parser::NUM_REF_FRAMES; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; @@ -35,9 +38,6 @@ use crate::decoder::DecoderEvent; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessBackendError; -use super::StatelessDecoderBackendPicture; - /// Stateless backend methods specific to VP9. pub trait StatelessVp9DecoderBackend: StatelessDecoderBackend + StatelessDecoderBackendPicture @@ -46,7 +46,7 @@ pub trait StatelessVp9DecoderBackend: fn new_sequence(&mut self, header: &Header) -> StatelessBackendResult<()>; /// Allocate all resources required to process a new picture. - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult; + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult; /// Called when the decoder wants the backend to finish the decoding /// operations for `picture`. @@ -281,12 +281,12 @@ where self.backend .new_picture(timestamp) .map_err(|e| match e { - StatelessBackendError::OutOfResources => { + NewPictureError::OutOfOutputBuffers => { DecodeError::NotEnoughOutputBuffers( num_required_pictures - i, ) } - e => DecodeError::BackendError(e), + e => DecodeError::from(e), }) .map(|picture| (frame, Some(picture))) } diff --git a/src/decoder/stateless/vp9/dummy.rs b/src/decoder/stateless/vp9/dummy.rs index d39ce14a..6a7ac6bd 100644 --- a/src/decoder/stateless/vp9/dummy.rs +++ b/src/decoder/stateless/vp9/dummy.rs @@ -16,6 +16,7 @@ use crate::codec::vp9::parser::NUM_REF_FRAMES; use crate::decoder::stateless::vp9::Segmentation; use crate::decoder::stateless::vp9::StatelessVp9DecoderBackend; use crate::decoder::stateless::vp9::Vp9; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; @@ -26,7 +27,7 @@ impl StatelessVp9DecoderBackend for Backend { Ok(()) } - fn new_picture(&mut self, _: u64) -> StatelessBackendResult { + fn new_picture(&mut self, _: u64) -> NewPictureResult { Ok(()) } diff --git a/src/decoder/stateless/vp9/vaapi.rs b/src/decoder/stateless/vp9/vaapi.rs index 603dfe2b..d314da52 100644 --- a/src/decoder/stateless/vp9/vaapi.rs +++ b/src/decoder/stateless/vp9/vaapi.rs @@ -27,8 +27,9 @@ use crate::codec::vp9::parser::NUM_REF_FRAMES; use crate::decoder::stateless::vp9::Segmentation; use crate::decoder::stateless::vp9::StatelessVp9DecoderBackend; use crate::decoder::stateless::vp9::Vp9; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -246,11 +247,11 @@ impl StatelessVp9DecoderBackend for VaapiB self.new_sequence(header, PoolCreationMode::Highest) } - fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult { + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult { let highest_pool = self.highest_pool(); let surface = highest_pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; let context = &metadata.context;