diff --git a/src/codec/h265/picture.rs b/src/codec/h265/picture.rs index a2a40116..d64618b2 100644 --- a/src/codec/h265/picture.rs +++ b/src/codec/h265/picture.rs @@ -53,7 +53,6 @@ impl PictureData { first_picture_after_eos: bool, prev_tid0_pic: Option<&PictureData>, max_pic_order_cnt_lsb: i32, - _timestamp: u64, ) -> Self { let hdr = &slice.header; let nalu_type = slice.nalu.header.type_; diff --git a/src/decoder/stateless.rs b/src/decoder/stateless.rs index 328d3e62..79b7ab16 100644 --- a/src/decoder/stateless.rs +++ b/src/decoder/stateless.rs @@ -40,11 +40,28 @@ use crate::decoder::StreamInfo; use crate::DecodedFormat; use crate::Resolution; +/// Error returned by `new_picture` methods of the backend, usually to indicate which kind of +/// resource is needed before the picture can be successfully created. +#[derive(Error, Debug)] +pub enum NewPictureError { + /// Indicates that the backend needs one output buffer to be returned to the pool before it can + /// proceed. + #[error("need one output buffer to be returned before operation can proceed")] + OutOfOutputBuffers, + /// No frame pool could satisfy the frame requirements. This indicate either an unhandled DRC + /// or an invalid stream. + #[error("no frame pool can satisfy the requested frame resolution {0:?}")] + NoFramePool(Resolution), + /// An unrecoverable backend error has occured. + #[error(transparent)] + BackendError(#[from] anyhow::Error), +} + +pub type NewPictureResult = Result; + /// Error returned by stateless backend methods. #[derive(Error, Debug)] pub enum StatelessBackendError { - #[error("not enough resources to proceed with the operation now")] - OutOfResources, #[error(transparent)] Other(#[from] anyhow::Error), } @@ -85,6 +102,21 @@ pub enum DecodeError { BackendError(#[from] StatelessBackendError), } +/// Convenience conversion for codecs that process a single frame per decode call. +impl From for DecodeError { + fn from(err: NewPictureError) -> Self { + match err { + NewPictureError::OutOfOutputBuffers => DecodeError::NotEnoughOutputBuffers(1), + e @ NewPictureError::NoFramePool(_) => { + DecodeError::BackendError(StatelessBackendError::Other(anyhow::anyhow!(e))) + } + NewPictureError::BackendError(e) => { + DecodeError::BackendError(StatelessBackendError::Other(e)) + } + } + } +} + mod private { use super::*; diff --git a/src/decoder/stateless/av1.rs b/src/decoder/stateless/av1.rs index 5d73475f..d8f779c8 100644 --- a/src/decoder/stateless/av1.rs +++ b/src/decoder/stateless/av1.rs @@ -7,7 +7,6 @@ use std::os::fd::BorrowedFd; use std::rc::Rc; use anyhow::anyhow; -use anyhow::Context; use crate::codec::av1::parser::FrameHeaderObu; use crate::codec::av1::parser::FrameObu; @@ -16,12 +15,11 @@ use crate::codec::av1::parser::ObuType; use crate::codec::av1::parser::ParsedObu; use crate::codec::av1::parser::Parser; use crate::codec::av1::parser::SequenceHeaderObu; -use crate::codec::av1::parser::NUM_REF_FRAMES; -use crate::Resolution; - use crate::codec::av1::parser::TileGroupObu; +use crate::codec::av1::parser::NUM_REF_FRAMES; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; @@ -31,8 +29,8 @@ use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; -use crate::decoder::FramePool; use crate::decoder::PoolLayer; +use crate::Resolution; #[cfg(test)] mod dummy; @@ -53,15 +51,23 @@ pub trait StatelessAV1DecoderBackend: highest_spatial_layer: Option, ) -> StatelessBackendResult<()>; - /// Called when the decoder determines that a new picture was found. + /// Called when the decoder determines that a new picture was found. The backend allocates all + /// the resources it needs to process that picture. fn new_picture( &mut self, - sequence: &SequenceHeaderObu, - picture: &FrameHeaderObu, + hdr: &FrameHeaderObu, timestamp: u64, - reference_frames: &[Option; NUM_REF_FRAMES], highest_spatial_layer: Option, - ) -> StatelessBackendResult; + ) -> NewPictureResult; + + /// Called to set the global parameters of a picture. + fn begin_picture( + &mut self, + picture: &mut Self::Picture, + sequence: &SequenceHeaderObu, + hdr: &FrameHeaderObu, + reference_frames: &[Option; NUM_REF_FRAMES], + ) -> StatelessBackendResult<()>; /// Called to dispatch a decode operation to the backend. #[allow(clippy::too_many_arguments)] @@ -157,34 +163,11 @@ where B: StatelessAV1DecoderBackend, B::Handle: Clone, { - fn count_frames(&mut self, bitstream: &[u8]) -> usize { - let mut nframes = 0; - let mut consumed = 0; - - while let Ok(obu) = self.codec.parser.parse_obu(&bitstream[consumed..]) { - let obu = match obu { - ParsedObu::Process(obu) => obu, - ParsedObu::Drop(length) => { - consumed += usize::try_from(length).unwrap(); - continue; - } - }; - - if matches!(obu.header.obu_type, ObuType::Frame | ObuType::FrameHeader) { - nframes += 1; - } - - consumed += obu.data.len(); - } - - nframes - } - fn decode_frame_header( &mut self, frame_header: FrameHeaderObu, timestamp: u64, - ) -> anyhow::Result<()> { + ) -> Result<(), DecodeError> { log::debug!( "Processing frame {} with timestamp {}", self.codec.frame_count, @@ -201,14 +184,19 @@ where handle: ref_frame.clone(), }); } else if let Some(sequence) = &self.codec.sequence { - let backend_picture = self.backend.new_picture( - sequence, + let mut backend_picture = self.backend.new_picture( &frame_header, timestamp, - &self.codec.reference_frames, self.codec.highest_spatial_layer, )?; + self.backend.begin_picture( + &mut backend_picture, + sequence, + &frame_header, + &self.codec.reference_frames, + )?; + self.codec.current_pic = Some(CurrentPicState::RegularFrame { header: frame_header.clone(), backend_picture, @@ -239,7 +227,7 @@ where Ok(()) } - fn decode_frame(&mut self, frame: FrameObu, timestamp: u64) -> anyhow::Result<()> { + fn decode_frame(&mut self, frame: FrameObu, timestamp: u64) -> Result<(), DecodeError> { let FrameObu { header, tile_group } = frame; self.decode_frame_header(header, timestamp)?; self.decode_tile_group(tile_group)?; @@ -327,173 +315,146 @@ where type Handle = B::Handle; type FramePool = B::FramePool; - fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { - let mut consumed = 0; - - let nframes = self.count_frames(bitstream); - /* we do not know the resolution at this point, as we haven't parsed the - * frames yet. Be conservative and check whether we have enough frames - * across all layers */ - let num_free_frames = self - .backend - .frame_pool(PoolLayer::All) - .iter() - .map(|x| x.num_free_frames()) - .min() - .ok_or(anyhow!("No pool found"))?; - - if matches!(self.decoding_state, DecodingState::Decoding) && num_free_frames < nframes { - return Err(DecodeError::NotEnoughOutputBuffers( - nframes - num_free_frames, - )); - } - - while let Ok(obu) = self.codec.parser.parse_obu(&bitstream[consumed..]) { - let obu = match obu { - ParsedObu::Process(obu) => obu, - // This OBU should be dropped. - ParsedObu::Drop(length) => { - consumed += usize::try_from(length).context("OBU length too large")?; - continue; - } - }; - - let obu_length = obu.data.len(); + /// Decode an AV1 stream. + /// + /// `bitstream` should initially be submitted as a whole temporal unit, however a call to this + /// method will only consume a single OBU. The caller must be careful to check the return value + /// and resubmit the remainder if the whole bitstream has not been consumed. + fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { + let obu = match self.codec.parser.parse_obu(bitstream)? { + ParsedObu::Process(obu) => obu, + // This OBU should be dropped. + ParsedObu::Drop(length) => return Ok(length as usize), + }; + let obu_length = obu.data.len(); - let is_decode_op = matches!( - obu.header.obu_type, - ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup - ); + let is_decode_op = matches!( + obu.header.obu_type, + ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup + ); - if is_decode_op { - match self.decoding_state { - /* we want to be here */ - DecodingState::Decoding => (), + if is_decode_op { + match self.decoding_state { + /* we want to be here */ + DecodingState::Decoding => (), - /* otherwise... */ - DecodingState::AwaitingStreamInfo => { - /* Skip input until we get information from the stream. */ - consumed += obu_length; - continue; - } - /* Ask the client to confirm the format before we can process this. */ - DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), - DecodingState::Reset => { - let mut parser = self.codec.parser.clone(); - - let is_key_frame = match obu.header.obu_type { - ObuType::Frame => { - let frame = parser.parse_frame_obu(obu.clone())?; - frame.header.frame_type == FrameType::KeyFrame - } - ObuType::FrameHeader => { - let fh = parser.parse_frame_header_obu(&obu)?; - fh.frame_type == FrameType::KeyFrame - } - _ => false, - }; - - /* we can only resume from key frames */ - if !is_key_frame { - consumed += obu_length; - continue; - } else { - self.decoding_state = DecodingState::Decoding; + /* otherwise... */ + DecodingState::AwaitingStreamInfo => { + /* Skip input until we get information from the stream. */ + return Ok(obu_length); + } + /* Ask the client to confirm the format before we can process this. */ + DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), + DecodingState::Reset => { + let mut parser = self.codec.parser.clone(); + + let is_key_frame = match obu.header.obu_type { + ObuType::Frame => { + let frame = parser.parse_frame_obu(obu.clone())?; + frame.header.frame_type == FrameType::KeyFrame } + ObuType::FrameHeader => { + let fh = parser.parse_frame_header_obu(&obu)?; + fh.frame_type == FrameType::KeyFrame + } + _ => false, + }; + + /* we can only resume from key frames */ + if !is_key_frame { + return Ok(obu_length); + } else { + self.decoding_state = DecodingState::Decoding; } } } + } - match obu.header.obu_type { - ObuType::SequenceHeader => { - let sequence = self.codec.parser.parse_sequence_header_obu(&obu)?; - let sequence_differs = match &self.codec.sequence { - Some(old_sequence) => **old_sequence != *sequence, - None => true, - }; - - if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) - || sequence_differs - { - if self.codec.current_pic.is_some() { - return Err(DecodeError::DecoderError(anyhow!( - "Broken stream: a picture is being decoded while a new sequence header is encountered" - ))); - } + /* We are in `Decoding` state if we reached here */ - /* make sure we sync *before* we clear any state in the backend */ - for f in &mut self.ready_queue.queue { - /* TODO: this fixes av1-1-b8-03-sizeup on Intel - * gen12, but we apparently do not do the same in - * VP9. How is it that we do not get similar crashes there? - * - * TODO: syncing before calling new_sequence() in VP9 may fix some tests - */ - f.sync()?; - } + match obu.header.obu_type { + ObuType::SequenceHeader => { + let sequence = self.codec.parser.parse_sequence_header_obu(&obu)?; + let sequence_differs = match &self.codec.sequence { + Some(old_sequence) => **old_sequence != *sequence, + None => true, + }; - log::debug!( - "Found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}", - Resolution::from(( - sequence.max_frame_width_minus_1 as u32 + 1, - sequence.max_frame_height_minus_1 as u32 + 1 - )), - sequence.seq_profile, - sequence.bit_depth - ); - /* there is nothing to drain, much like vp8 and vp9 */ - self.codec.highest_spatial_layer = - self.codec.parser.highest_operating_point(); - self.backend - .new_sequence(&sequence, self.codec.highest_spatial_layer)?; - self.await_format_change(sequence); - } - } - ObuType::TemporalDelimiter => { - self.codec.parser.parse_temporal_delimiter_obu(&obu)? - } - ObuType::FrameHeader => { + if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) + || sequence_differs + { if self.codec.current_pic.is_some() { - /* submit this frame immediately, as we need to update the - * DPB and the reference info state *before* processing the - * next frame */ - self.submit_frame(timestamp)?; + return Err(DecodeError::DecoderError(anyhow!( + "broken stream: a picture is being decoded while a new sequence header is encountered" + ))); } - let frame_header = self.codec.parser.parse_frame_header_obu(&obu)?; - self.decode_frame_header(frame_header, timestamp)?; - } - ObuType::TileGroup => { - let tile_group = self.codec.parser.parse_tile_group_obu(obu)?; - self.decode_tile_group(tile_group)?; + + /* make sure we sync *before* we clear any state in the backend */ + for f in &mut self.ready_queue.queue { + /* TODO: this fixes av1-1-b8-03-sizeup on Intel + * gen12, but we apparently do not do the same in + * VP9. How is it that we do not get similar crashes there? + * + * TODO: syncing before calling new_sequence() in VP9 may fix some tests + */ + f.sync()?; + } + + log::debug!( + "found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}", + Resolution::from(( + sequence.max_frame_width_minus_1 as u32 + 1, + sequence.max_frame_height_minus_1 as u32 + 1 + )), + sequence.seq_profile, + sequence.bit_depth + ); + /* there is nothing to drain, much like vp8 and vp9 */ + self.codec.highest_spatial_layer = self.codec.parser.highest_operating_point(); + self.backend + .new_sequence(&sequence, self.codec.highest_spatial_layer)?; + self.await_format_change(sequence); } - ObuType::Frame => { - let frame = self.codec.parser.parse_frame_obu(obu)?; - self.decode_frame(frame, timestamp)?; + } + ObuType::TemporalDelimiter => self.codec.parser.parse_temporal_delimiter_obu(&obu)?, + ObuType::FrameHeader => { + if self.codec.current_pic.is_some() { /* submit this frame immediately, as we need to update the * DPB and the reference info state *before* processing the * next frame */ self.submit_frame(timestamp)?; } - ObuType::TileList => { - return Err(DecodeError::DecoderError(anyhow!( - "Large tile scale mode is not supported" - ))); - } - other => { - log::debug!("Skipping OBU of type {:?}", other); - } + let frame_header = self.codec.parser.parse_frame_header_obu(&obu)?; + self.decode_frame_header(frame_header, timestamp)?; + } + ObuType::TileGroup => { + let tile_group = self.codec.parser.parse_tile_group_obu(obu)?; + self.decode_tile_group(tile_group)?; + } + ObuType::Frame => { + let frame = self.codec.parser.parse_frame_obu(obu)?; + self.decode_frame(frame, timestamp)?; + /* submit this frame immediately, as we need to update the + * DPB and the reference info state *before* processing the + * next frame */ + self.submit_frame(timestamp)?; + } + ObuType::TileList => { + return Err(DecodeError::DecoderError(anyhow!( + "large tile scale mode is not supported" + ))); + } + other => { + log::debug!("skipping OBU of type {:?}", other); } - - consumed += obu_length; } - /* we may already have dispatched work if we got ObuType::Frame */ - if self.codec.current_pic.is_some() { - /* dispatch work to the backend */ + /* Submit the last frame if we have reached the end of the temporal unit. */ + if bitstream.len() == obu_length && self.codec.current_pic.is_some() { self.submit_frame(timestamp)?; } - Ok(consumed) + Ok(obu_length) } fn flush(&mut self) -> Result<(), super::DecodeError> { diff --git a/src/decoder/stateless/av1/dummy.rs b/src/decoder/stateless/av1/dummy.rs index d87dfe41..9938ed7a 100644 --- a/src/decoder/stateless/av1/dummy.rs +++ b/src/decoder/stateless/av1/dummy.rs @@ -28,12 +28,20 @@ impl StatelessAV1DecoderBackend for Backend { fn new_picture( &mut self, - _: &crate::codec::av1::parser::SequenceHeaderObu, _: &crate::codec::av1::parser::FrameHeaderObu, _: u64, - _: &[Option; crate::codec::av1::parser::NUM_REF_FRAMES], _: Option, - ) -> crate::decoder::stateless::StatelessBackendResult { + ) -> crate::decoder::stateless::NewPictureResult { + Ok(()) + } + + fn begin_picture( + &mut self, + _: &mut Self::Picture, + _: &crate::codec::av1::parser::SequenceHeaderObu, + _: &crate::codec::av1::parser::FrameHeaderObu, + _: &[Option; crate::codec::av1::parser::NUM_REF_FRAMES], + ) -> crate::decoder::stateless::StatelessBackendResult<()> { Ok(()) } diff --git a/src/decoder/stateless/av1/vaapi.rs b/src/decoder/stateless/av1/vaapi.rs index aed31961..80be00e6 100644 --- a/src/decoder/stateless/av1/vaapi.rs +++ b/src/decoder/stateless/av1/vaapi.rs @@ -29,8 +29,10 @@ use crate::codec::av1::parser::NUM_REF_FRAMES; use crate::codec::av1::parser::SEG_LVL_MAX; use crate::decoder::stateless::av1::Av1; use crate::decoder::stateless::av1::StatelessAV1DecoderBackend; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; +use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::BlockingMode; @@ -507,7 +509,7 @@ impl StatelessAV1DecoderBackend for VaapiB &mut self, sequence: &Rc, highest_spatial_layer: Option, - ) -> crate::decoder::stateless::StatelessBackendResult<()> { + ) -> StatelessBackendResult<()> { let pool_creation_mode = match highest_spatial_layer { Some(highest_layer) => { /* The spec mandates a 2:1 or 1.5:1 ratio, let's go with 2:1 to @@ -526,42 +528,45 @@ impl StatelessAV1DecoderBackend for VaapiB fn new_picture( &mut self, - sequence: &SequenceHeaderObu, hdr: &FrameHeaderObu, timestamp: u64, - reference_frames: &[Option; NUM_REF_FRAMES], highest_spatial_layer: Option, - ) -> crate::decoder::stateless::StatelessBackendResult { - let surface = match highest_spatial_layer { + ) -> NewPictureResult { + let pool = match highest_spatial_layer { Some(_) => { let layer = Resolution { width: hdr.upscaled_width, height: hdr.frame_height, }; - let pool = self - .pool(layer) - .ok_or(StatelessBackendError::Other(anyhow!( - "No pool available for this layer" - )))?; - - pool.get_surface() - .ok_or(StatelessBackendError::OutOfResources)? - } - None => { - let highest_pool = self.highest_pool(); - highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)? + self.pool(layer) + .ok_or(NewPictureError::NoFramePool(layer))? } + None => self.highest_pool(), }; + let surface = pool + .get_surface() + .ok_or(NewPictureError::OutOfOutputBuffers)?; + let metadata = self.metadata_state.get_parsed()?; - let mut picture = VaPicture::new(timestamp, Rc::clone(&metadata.context), surface); + Ok(VaPicture::new( + timestamp, + Rc::clone(&metadata.context), + surface, + )) + } - let surface_id = picture.surface().id(); + fn begin_picture( + &mut self, + picture: &mut Self::Picture, + sequence: &SequenceHeaderObu, + hdr: &FrameHeaderObu, + reference_frames: &[Option; NUM_REF_FRAMES], + ) -> StatelessBackendResult<()> { + let metadata = self.metadata_state.get_parsed()?; - let pic_param = build_pic_param(hdr, sequence, surface_id, reference_frames) + let pic_param = build_pic_param(hdr, sequence, picture.surface().id(), reference_frames) .context("Failed to build picture parameter")?; let pic_param = metadata .context @@ -569,7 +574,7 @@ impl StatelessAV1DecoderBackend for VaapiB .context("Failed to create picture parameter buffer")?; picture.add_buffer(pic_param); - Ok(picture) + Ok(()) } fn decode_tile_group( @@ -598,10 +603,7 @@ impl StatelessAV1DecoderBackend for VaapiB Ok(()) } - fn submit_picture( - &mut self, - picture: Self::Picture, - ) -> crate::decoder::stateless::StatelessBackendResult { + fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult { self.process_picture::(picture) } } diff --git a/src/decoder/stateless/h264.rs b/src/decoder/stateless/h264.rs index 1aa69e4a..9fc88dc6 100644 --- a/src/decoder/stateless/h264.rs +++ b/src/decoder/stateless/h264.rs @@ -40,22 +40,21 @@ use crate::codec::h264::picture::RcPictureData; use crate::codec::h264::picture::Reference; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - fn get_raster_from_zigzag_8x8(src: [u8; 64], dst: &mut [u8; 64]) { const ZIGZAG_8X8: [usize; 64] = [ 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, @@ -84,11 +83,7 @@ pub trait StatelessH264DecoderBackend: fn new_sequence(&mut self, sps: &Rc) -> StatelessBackendResult<()>; /// Called when the decoder determines that a frame or field was found. - fn new_picture( - &mut self, - picture: &PictureData, - timestamp: u64, - ) -> StatelessBackendResult; + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult; /// Called when the decoder determines that a second field was found. /// Indicates that the underlying BackendHandle is to be shared between the @@ -96,10 +91,9 @@ pub trait StatelessH264DecoderBackend: /// resource and can thus be presented together as a single frame. fn new_field_picture( &mut self, - picture: &PictureData, timestamp: u64, first_field: &Self::Handle, - ) -> StatelessBackendResult; + ) -> NewPictureResult; /// Called by the decoder when starting a new frame or field. fn start_picture( @@ -1098,6 +1092,14 @@ where timestamp: u64, slice: &Slice, ) -> Result, DecodeError> { + // Start by securing the backend picture before modifying our state. + let first_field = self.codec.find_first_field(&slice.header)?; + let mut backend_pic = if let Some(first_field) = &first_field { + self.backend.new_field_picture(timestamp, &first_field.1) + } else { + self.backend.new_picture(timestamp) + }?; + let nalu_hdr = &slice.nalu.header; if nalu_hdr.idr_pic_flag { @@ -1120,17 +1122,6 @@ where return Err(DecodeError::CheckEvents); } - if self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(anyhow!("No pool found"))? - .num_free_frames() - == 0 - { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } - let current_macroblock = match pps.sps.separate_colour_plane_flag { true => CurrentMacroblockTracking::SeparateColorPlane(Default::default()), false => CurrentMacroblockTracking::NonSeparateColorPlane(0), @@ -1142,7 +1133,6 @@ where self.handle_frame_num_gap(&pps.sps, frame_num, timestamp)?; } - let first_field = self.codec.find_first_field(&slice.header)?; let pic = self.init_current_pic( slice, &pps.sps, @@ -1153,13 +1143,6 @@ where debug!("Decode picture POC {:?}", pic.pic_order_cnt); - let mut backend_pic = if let Some(first_field) = first_field { - self.backend - .new_field_picture(&pic, timestamp, &first_field.1)? - } else { - self.backend.new_picture(&pic, timestamp)? - }; - self.backend.start_picture( &mut backend_pic, &pic, diff --git a/src/decoder/stateless/h264/dummy.rs b/src/decoder/stateless/h264/dummy.rs index 90b31c82..b6e1d39c 100644 --- a/src/decoder/stateless/h264/dummy.rs +++ b/src/decoder/stateless/h264/dummy.rs @@ -19,6 +19,7 @@ use crate::codec::h264::parser::Sps; use crate::codec::h264::picture::PictureData; use crate::decoder::stateless::h264::StatelessH264DecoderBackend; use crate::decoder::stateless::h264::H264; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; @@ -41,12 +42,7 @@ impl StatelessH264DecoderBackend for Backend { Ok(()) } - fn new_field_picture( - &mut self, - _: &PictureData, - _: u64, - _: &Self::Handle, - ) -> StatelessBackendResult<()> { + fn new_field_picture(&mut self, _: u64, _: &Self::Handle) -> NewPictureResult<()> { Ok(()) } @@ -68,7 +64,7 @@ impl StatelessH264DecoderBackend for Backend { }) } - fn new_picture(&mut self, _: &PictureData, _: u64) -> StatelessBackendResult<()> { + fn new_picture(&mut self, _: u64) -> NewPictureResult<()> { Ok(()) } } diff --git a/src/decoder/stateless/h264/vaapi.rs b/src/decoder/stateless/h264/vaapi.rs index ee1a0a62..f909f501 100644 --- a/src/decoder/stateless/h264/vaapi.rs +++ b/src/decoder/stateless/h264/vaapi.rs @@ -35,8 +35,9 @@ use crate::codec::h264::picture::PictureData; use crate::codec::h264::picture::Reference; use crate::decoder::stateless::h264::StatelessH264DecoderBackend; use crate::decoder::stateless::h264::H264; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -540,15 +541,11 @@ impl StatelessH264DecoderBackend for Vaapi self.process_picture::(picture) } - fn new_picture( - &mut self, - _: &PictureData, - timestamp: u64, - ) -> StatelessBackendResult { + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult { let highest_pool = self.highest_pool(); let surface = highest_pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; @@ -561,10 +558,9 @@ impl StatelessH264DecoderBackend for Vaapi fn new_field_picture( &mut self, - _: &PictureData, timestamp: u64, first_field: &Self::Handle, - ) -> StatelessBackendResult { + ) -> NewPictureResult { // Decode to the same surface as the first field picture. Ok(first_field .borrow() diff --git a/src/decoder/stateless/h265.rs b/src/decoder/stateless/h265.rs index 9cc8dbf9..d7398777 100644 --- a/src/decoder/stateless/h265.rs +++ b/src/decoder/stateless/h265.rs @@ -30,22 +30,21 @@ use crate::codec::h265::picture::PictureData; use crate::codec::h265::picture::Reference; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - const MAX_DPB_SIZE: usize = 16; // Equation 5-8 @@ -115,9 +114,9 @@ pub trait StatelessH265DecoderBackend: /// Called when the decoder determines that a frame or field was found. fn new_picture( &mut self, - picture: &PictureData, + coded_resolution: Resolution, timestamp: u64, - ) -> StatelessBackendResult; + ) -> NewPictureResult; /// Called by the decoder for every frame or field found. #[allow(clippy::too_many_arguments)] @@ -290,8 +289,6 @@ pub struct H265DecoderState { /// The current active SPS id. cur_sps_id: u8, - /// The current active PPS id. - cur_pps_id: u8, /// Used to identify first picture in decoding order or first picture that /// follows an EOS NALU. @@ -330,7 +327,6 @@ where rps: Default::default(), dpb: Default::default(), cur_sps_id: Default::default(), - cur_pps_id: Default::default(), first_picture_after_eos: true, first_picture_in_bitstream: true, prev_tid_0_pic: Default::default(), @@ -928,44 +924,25 @@ where timestamp: u64, slice: &Slice, ) -> Result>, DecodeError> { - let layer = PoolLayer::Layer(self.coded_resolution); - if self - .backend - .frame_pool(layer) - .pop() - .ok_or(anyhow!("Pool not found"))? - .num_free_frames() - == 0 - { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } - - let pps = self - .codec - .parser - .get_pps(slice.header.pic_parameter_set_id) - .context("Invalid PPS in handle_picture")?; - - let pps_id = pps.pic_parameter_set_id; - self.update_current_set_ids(pps_id)?; + self.update_current_set_ids(slice.header.pic_parameter_set_id)?; self.renegotiate_if_needed(RenegotiationType::CurrentSps)?; - // We renegotiated and must return the NALU and wait. if matches!(self.decoding_state, DecodingState::AwaitingFormat(_)) { return Err(DecodeError::CheckEvents); } + let mut backend_pic = self.backend.new_picture(self.coded_resolution, timestamp)?; + let pic = PictureData::new_from_slice( slice, self.codec .parser - .get_pps(self.codec.cur_pps_id) + .get_pps(slice.header.pic_parameter_set_id) .context("Invalid PPS")?, self.codec.first_picture_in_bitstream, self.codec.first_picture_after_eos, self.codec.prev_tid_0_pic.as_ref(), self.codec.max_pic_order_cnt_lsb, - timestamp, ); self.codec.first_picture_after_eos = false; @@ -994,19 +971,21 @@ where self.decode_rps(slice, &pic)?; self.update_dpb_before_decoding(&pic)?; - let mut backend_pic = self.backend.new_picture(&pic, timestamp)?; - + let pps = self + .codec + .parser + .get_pps(slice.header.pic_parameter_set_id) + .context("invalid PPS ID")?; + let sps = self + .codec + .parser + .get_sps(pps.seq_parameter_set_id) + .context("invalid SPS ID")?; self.backend.begin_picture( &mut backend_pic, &pic, - self.codec - .parser - .get_sps(self.codec.cur_sps_id) - .context("Invalid SPS")?, - self.codec - .parser - .get_pps(self.codec.cur_pps_id) - .context("Invalid PPS")?, + sps, + pps, &self.codec.dpb, &self.codec.rps, slice, @@ -1022,7 +1001,6 @@ where fn update_current_set_ids(&mut self, pps_id: u8) -> anyhow::Result<()> { let pps = self.codec.parser.get_pps(pps_id).context("Invalid PPS")?; - self.codec.cur_pps_id = pps.pic_parameter_set_id; self.codec.cur_sps_id = pps.seq_parameter_set_id; Ok(()) } @@ -1053,17 +1031,21 @@ where pic.ref_pic_lists = self.build_ref_pic_lists(&slice.header, &pic.pic)?; + let pps = self + .codec + .parser + .get_pps(slice.header.pic_parameter_set_id) + .context("invalid PPS ID")?; + let sps = self + .codec + .parser + .get_sps(pps.seq_parameter_set_id) + .context("invalid SPS ID")?; self.backend.decode_slice( &mut pic.backend_pic, slice, - self.codec - .parser - .get_sps(self.codec.cur_sps_id) - .context("Invalid SPS id")?, - self.codec - .parser - .get_pps(self.codec.cur_pps_id) - .context("Invalid PPS id")?, + sps, + pps, &pic.ref_pic_lists.ref_pic_list0, &pic.ref_pic_lists.ref_pic_list1, )?; diff --git a/src/decoder/stateless/h265/dummy.rs b/src/decoder/stateless/h265/dummy.rs index 98cf744c..c0bd12e6 100644 --- a/src/decoder/stateless/h265/dummy.rs +++ b/src/decoder/stateless/h265/dummy.rs @@ -16,6 +16,7 @@ use crate::decoder::stateless::StatelessDecoder; use crate::decoder::BlockingMode; use crate::decoder::stateless::h265::StatelessH265DecoderBackend; +use crate::Resolution; impl StatelessH265DecoderBackend for Backend { fn new_sequence( @@ -27,9 +28,9 @@ impl StatelessH265DecoderBackend for Backend { fn new_picture( &mut self, - _: &crate::codec::h265::picture::PictureData, + _: Resolution, _: u64, - ) -> crate::decoder::stateless::StatelessBackendResult { + ) -> crate::decoder::stateless::NewPictureResult { Ok(()) } diff --git a/src/decoder/stateless/h265/vaapi.rs b/src/decoder/stateless/h265/vaapi.rs index 43e5a191..d2c3f92c 100644 --- a/src/decoder/stateless/h265/vaapi.rs +++ b/src/decoder/stateless/h265/vaapi.rs @@ -37,12 +37,16 @@ use crate::decoder::stateless::h265::RefPicListEntry; use crate::decoder::stateless::h265::RefPicSet; use crate::decoder::stateless::h265::StatelessH265DecoderBackend; use crate::decoder::stateless::h265::H265; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; +use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; +use crate::decoder::stateless::StatelessDecoderBackend; use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::BlockingMode; +use crate::Resolution; enum ScalingListType { Sps, @@ -596,13 +600,17 @@ impl StatelessH265DecoderBackend for Vaapi fn new_picture( &mut self, - _: &PictureData, + coded_resolution: Resolution, timestamp: u64, - ) -> StatelessBackendResult { - let highest_pool = self.highest_pool(); - let surface = highest_pool + ) -> NewPictureResult { + let layer = PoolLayer::Layer(coded_resolution); + let pool = self + .frame_pool(layer) + .pop() + .ok_or(NewPictureError::NoFramePool(coded_resolution))?; + let surface = pool .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; + .ok_or(NewPictureError::OutOfOutputBuffers)?; let metadata = self.metadata_state.get_parsed()?; Ok(VaapiH265Picture { diff --git a/src/decoder/stateless/vp8.rs b/src/decoder/stateless/vp8.rs index 2190441e..d3b5caf6 100644 --- a/src/decoder/stateless/vp8.rs +++ b/src/decoder/stateless/vp8.rs @@ -10,8 +10,6 @@ mod vaapi; use std::os::fd::AsFd; use std::os::fd::BorrowedFd; -use anyhow::anyhow; - use crate::codec::vp8::parser::Frame; use crate::codec::vp8::parser::Header; use crate::codec::vp8::parser::MbLfAdjustments; @@ -19,22 +17,21 @@ use crate::codec::vp8::parser::Parser; use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - /// Stateless backend methods specific to VP8. pub trait StatelessVp8DecoderBackend: StatelessDecoderBackend + StatelessDecoderBackendPicture @@ -42,6 +39,9 @@ pub trait StatelessVp8DecoderBackend: /// Called when new stream parameters are found. fn new_sequence(&mut self, header: &Header) -> StatelessBackendResult<()>; + /// Called when the decoder determines that a frame or field was found. + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult; + /// Called when the decoder wants the backend to finish the decoding /// operations for `picture`. /// @@ -50,14 +50,14 @@ pub trait StatelessVp8DecoderBackend: #[allow(clippy::too_many_arguments)] fn submit_picture( &mut self, - picture: &Header, + picture: Self::Picture, + hdr: &Header, last_ref: &Option, golden_ref: &Option, alt_ref: &Option, bitstream: &[u8], segmentation: &Segmentation, mb_lf_adjust: &MbLfAdjustments, - timestamp: u64, ) -> StatelessBackendResult; } @@ -178,20 +178,11 @@ where { /// Handle a single frame. fn handle_frame(&mut self, frame: Frame, timestamp: u64) -> Result<(), DecodeError> { - if self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? - .num_free_frames() - == 0 - { - return Err(DecodeError::NotEnoughOutputBuffers(1)); - } - let show_frame = frame.header.show_frame; + let picture = self.backend.new_picture(timestamp)?; let decoded_handle = self.backend.submit_picture( + picture, &frame.header, &self.codec.last_picture, &self.codec.golden_ref_picture, @@ -199,7 +190,6 @@ where frame.as_ref(), self.codec.parser.segmentation(), self.codec.parser.mb_lf_adjust(), - timestamp, )?; if self.blocking_mode == BlockingMode::Blocking { diff --git a/src/decoder/stateless/vp8/dummy.rs b/src/decoder/stateless/vp8/dummy.rs index 3a3da5a0..c4a1f4f0 100644 --- a/src/decoder/stateless/vp8/dummy.rs +++ b/src/decoder/stateless/vp8/dummy.rs @@ -15,6 +15,7 @@ use crate::codec::vp8::parser::MbLfAdjustments; use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::vp8::StatelessVp8DecoderBackend; use crate::decoder::stateless::vp8::Vp8; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; @@ -25,8 +26,13 @@ impl StatelessVp8DecoderBackend for Backend { Ok(()) } + fn new_picture(&mut self, _: u64) -> NewPictureResult { + Ok(()) + } + fn submit_picture( &mut self, + _: Self::Picture, _: &Header, _: &Option, _: &Option, @@ -34,7 +40,6 @@ impl StatelessVp8DecoderBackend for Backend { _: &[u8], _: &Segmentation, _: &MbLfAdjustments, - _: u64, ) -> StatelessBackendResult { Ok(Handle { handle: Rc::new(RefCell::new(Default::default())), diff --git a/src/decoder/stateless/vp8/vaapi.rs b/src/decoder/stateless/vp8/vaapi.rs index 119fed1d..a6187c4e 100644 --- a/src/decoder/stateless/vp8/vaapi.rs +++ b/src/decoder/stateless/vp8/vaapi.rs @@ -18,13 +18,15 @@ use crate::backend::vaapi::decoder::va_surface_id; use crate::backend::vaapi::decoder::PoolCreationMode; use crate::backend::vaapi::decoder::VaStreamInfo; use crate::backend::vaapi::decoder::VaapiBackend; +use crate::backend::vaapi::decoder::VaapiPicture; use crate::codec::vp8::parser::Header; use crate::codec::vp8::parser::MbLfAdjustments; use crate::codec::vp8::parser::Segmentation; use crate::decoder::stateless::vp8::StatelessVp8DecoderBackend; use crate::decoder::stateless::vp8::Vp8; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -208,7 +210,7 @@ fn build_slice_param(frame_hdr: &Header, slice_size: usize) -> anyhow::Result
  • StatelessDecoderBackendPicture for VaapiBackend { - type Picture = (); + type Picture = VaapiPicture; } impl StatelessVp8DecoderBackend for VaapiBackend { @@ -216,40 +218,52 @@ impl StatelessVp8DecoderBackend for VaapiB self.new_sequence(header, PoolCreationMode::Highest) } + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult { + let highest_pool = self.highest_pool(); + let surface = highest_pool + .get_surface() + .ok_or(NewPictureError::OutOfOutputBuffers)?; + + let metadata = self.metadata_state.get_parsed()?; + + Ok(VaPicture::new( + timestamp, + Rc::clone(&metadata.context), + surface, + )) + } + fn submit_picture( &mut self, - picture: &Header, + mut picture: Self::Picture, + hdr: &Header, last_ref: &Option, golden_ref: &Option, alt_ref: &Option, bitstream: &[u8], segmentation: &Segmentation, mb_lf_adjust: &MbLfAdjustments, - timestamp: u64, ) -> StatelessBackendResult { + let highest_pool = self.highest_pool(); let last_ref = va_surface_id(last_ref); let golden_ref = va_surface_id(golden_ref); let alt_ref = va_surface_id(alt_ref); - let highest_pool = self.highest_pool(); - let surface = highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; let coded_resolution = highest_pool.coded_resolution(); let metadata = self.metadata_state.get_parsed()?; let context = &metadata.context; let iq_buffer = context - .create_buffer(build_iq_matrix(picture, segmentation)?) + .create_buffer(build_iq_matrix(hdr, segmentation)?) .context("while creating IQ matrix buffer")?; let probs = context - .create_buffer(build_probability_table(picture)) + .create_buffer(build_probability_table(hdr)) .context("while creating probability table buffer")?; let pic_param = context .create_buffer(build_pic_param( - picture, + hdr, &coded_resolution, segmentation, mb_lf_adjust, @@ -260,23 +274,21 @@ impl StatelessVp8DecoderBackend for VaapiB .context("while creating pic params buffer")?; let slice_param = context - .create_buffer(build_slice_param(picture, bitstream.len())?) + .create_buffer(build_slice_param(hdr, bitstream.len())?) .context("while creating slice params buffer")?; let slice_data = context .create_buffer(libva::BufferType::SliceData(Vec::from(bitstream))) .context("while creating slice data buffer")?; - let mut va_picture = VaPicture::new(timestamp, Rc::clone(context), surface); - // Add buffers with the parsed data. - va_picture.add_buffer(iq_buffer); - va_picture.add_buffer(probs); - va_picture.add_buffer(pic_param); - va_picture.add_buffer(slice_param); - va_picture.add_buffer(slice_data); + picture.add_buffer(iq_buffer); + picture.add_buffer(probs); + picture.add_buffer(pic_param); + picture.add_buffer(slice_param); + picture.add_buffer(slice_data); - self.process_picture::(va_picture) + self.process_picture::(picture) } } diff --git a/src/decoder/stateless/vp9.rs b/src/decoder/stateless/vp9.rs index aaa118ea..89c1bf37 100644 --- a/src/decoder/stateless/vp9.rs +++ b/src/decoder/stateless/vp9.rs @@ -10,7 +10,6 @@ mod vaapi; use std::os::fd::AsFd; use std::os::fd::BorrowedFd; -use anyhow::anyhow; use log::debug; use crate::codec::vp9::parser::BitDepth; @@ -23,22 +22,22 @@ use crate::codec::vp9::parser::MAX_SEGMENTS; use crate::codec::vp9::parser::NUM_REF_FRAMES; use crate::decoder::stateless::DecodeError; use crate::decoder::stateless::DecodingState; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::PoolLayer; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessCodec; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::StatelessDecoderBackendPicture; use crate::decoder::stateless::StatelessVideoDecoder; use crate::decoder::stateless::TryFormat; use crate::decoder::BlockingMode; use crate::decoder::DecodedHandle; use crate::decoder::DecoderEvent; -use crate::decoder::FramePool; use crate::decoder::StreamInfo; use crate::Resolution; -use super::StatelessDecoderBackendPicture; - /// Stateless backend methods specific to VP9. pub trait StatelessVp9DecoderBackend: StatelessDecoderBackend + StatelessDecoderBackendPicture @@ -46,6 +45,9 @@ pub trait StatelessVp9DecoderBackend: /// Called when new stream parameters are found. fn new_sequence(&mut self, header: &Header) -> StatelessBackendResult<()>; + /// Allocate all resources required to process a new picture. + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult; + /// Called when the decoder wants the backend to finish the decoding /// operations for `picture`. /// @@ -53,10 +55,10 @@ pub trait StatelessVp9DecoderBackend: /// and then assign the ownership of the Picture to the Handle. fn submit_picture( &mut self, - picture: &Header, + picture: Self::Picture, + hdr: &Header, reference_frames: &[Option; NUM_REF_FRAMES], bitstream: &[u8], - timestamp: u64, segmentation: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult; } @@ -150,53 +152,53 @@ where Ok(()) } - /// Handle a single frame. - fn handle_frame(&mut self, frame: &Frame, timestamp: u64) -> Result<(), DecodeError> { - let decoded_handle = if frame.header.show_existing_frame { - // Frame to be shown. Because the spec mandates that frame_to_show_map_idx references a - // valid entry in the DPB, an non-existing index means that the stream is invalid. - let idx = usize::from(frame.header.frame_to_show_map_idx); - let ref_frame = self - .codec - .reference_frames - .get(idx) - .ok_or_else(|| anyhow::anyhow!("invalid reference frame index in header"))? - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!("empty reference frame referenced in frame header") - })?; - - // We are done, no further processing needed. - ref_frame.clone() - } else { - // Otherwise, we must actually arrange to decode a frame - let refresh_frame_flags = frame.header.refresh_frame_flags; - - Segmentation::update_segmentation(&mut self.codec.segmentation, &frame.header); - let decoded_handle = self.backend.submit_picture( - &frame.header, - &self.codec.reference_frames, - frame.as_ref(), - timestamp, - &self.codec.segmentation, - )?; - - if self.blocking_mode == BlockingMode::Blocking { - decoded_handle.sync()?; - } + /// Handle a frame which `show_existing_frame` flag is `true`. + fn handle_show_existing_frame(&mut self, frame_to_show_map_idx: u8) -> Result<(), DecodeError> { + // Frame to be shown. Because the spec mandates that frame_to_show_map_idx references a + // valid entry in the DPB, an non-existing index means that the stream is invalid. + let idx = usize::from(frame_to_show_map_idx); + let ref_frame = self + .codec + .reference_frames + .get(idx) + .ok_or_else(|| anyhow::anyhow!("invalid reference frame index in header"))? + .as_ref() + .ok_or_else(|| anyhow::anyhow!("empty reference frame referenced in frame header"))?; + + // We are done, no further processing needed. + let decoded_handle = ref_frame.clone(); + + self.ready_queue.push(decoded_handle); + + Ok(()) + } - // Do DPB management - Self::update_references( - &mut self.codec.reference_frames, - &decoded_handle, - refresh_frame_flags, - )?; + /// Decode a single frame. + fn handle_frame(&mut self, frame: &Frame, picture: B::Picture) -> Result<(), DecodeError> { + let refresh_frame_flags = frame.header.refresh_frame_flags; - decoded_handle - }; + Segmentation::update_segmentation(&mut self.codec.segmentation, &frame.header); - let show_existing_frame = frame.header.show_existing_frame; - if frame.header.show_frame || show_existing_frame { + let decoded_handle = self.backend.submit_picture( + picture, + &frame.header, + &self.codec.reference_frames, + frame.as_ref(), + &self.codec.segmentation, + )?; + + if self.blocking_mode == BlockingMode::Blocking { + decoded_handle.sync()?; + } + + // Do DPB management + Self::update_references( + &mut self.codec.reference_frames, + &decoded_handle, + refresh_frame_flags, + )?; + + if frame.header.show_frame { self.ready_queue.push(decoded_handle); } @@ -227,20 +229,6 @@ where fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result { let frames = self.codec.parser.parse_chunk(bitstream)?; - let num_free_frames = self - .backend - .frame_pool(PoolLayer::Highest) - .pop() - .ok_or(DecodeError::DecoderError(anyhow!("No pool found")))? - .num_free_frames(); - - if matches!(self.decoding_state, DecodingState::Decoding) && num_free_frames < frames.len() - { - return Err(DecodeError::NotEnoughOutputBuffers( - frames.len() - num_free_frames, - )); - } - // With SVC, the first frame will usually be a key-frame, with // inter-frames carrying the other layers. // @@ -272,13 +260,48 @@ where } } - for frame in frames { - match &mut self.decoding_state { - // Skip input until we get information from the stream. - DecodingState::AwaitingStreamInfo | DecodingState::Reset => (), - // Ask the client to confirm the format before we can process this. - DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), - DecodingState::Decoding => self.handle_frame(&frame, timestamp)?, + match &mut self.decoding_state { + // Skip input until we get information from the stream. + DecodingState::AwaitingStreamInfo | DecodingState::Reset => (), + // Ask the client to confirm the format before we can process this. + DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), + DecodingState::Decoding => { + // First allocate all the pictures we need for this superframe. + let num_required_pictures = frames + .iter() + .filter(|f| !f.header.show_existing_frame) + .count(); + let frames_with_pictures = frames + .into_iter() + .enumerate() + .map(|(i, frame)| { + if frame.header.show_existing_frame { + Ok((frame, None)) + } else { + self.backend + .new_picture(timestamp) + .map_err(|e| match e { + NewPictureError::OutOfOutputBuffers => { + DecodeError::NotEnoughOutputBuffers( + num_required_pictures - i, + ) + } + e => DecodeError::from(e), + }) + .map(|picture| (frame, Some(picture))) + } + }) + .collect::, _>>()?; + + // Then process each frame. + for (frame, picture) in frames_with_pictures { + match picture { + None => { + self.handle_show_existing_frame(frame.header.frame_to_show_map_idx)? + } + Some(picture) => self.handle_frame(&frame, picture)?, + } + } } } diff --git a/src/decoder/stateless/vp9/dummy.rs b/src/decoder/stateless/vp9/dummy.rs index a884635c..6a7ac6bd 100644 --- a/src/decoder/stateless/vp9/dummy.rs +++ b/src/decoder/stateless/vp9/dummy.rs @@ -16,6 +16,7 @@ use crate::codec::vp9::parser::NUM_REF_FRAMES; use crate::decoder::stateless::vp9::Segmentation; use crate::decoder::stateless::vp9::StatelessVp9DecoderBackend; use crate::decoder::stateless::vp9::Vp9; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; @@ -26,12 +27,16 @@ impl StatelessVp9DecoderBackend for Backend { Ok(()) } + fn new_picture(&mut self, _: u64) -> NewPictureResult { + Ok(()) + } + fn submit_picture( &mut self, + _: Self::Picture, _: &Header, _: &[Option; NUM_REF_FRAMES], _: &[u8], - _: u64, _: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult { Ok(Handle { diff --git a/src/decoder/stateless/vp9/vaapi.rs b/src/decoder/stateless/vp9/vaapi.rs index 54377c30..d314da52 100644 --- a/src/decoder/stateless/vp9/vaapi.rs +++ b/src/decoder/stateless/vp9/vaapi.rs @@ -15,6 +15,7 @@ use crate::backend::vaapi::decoder::va_surface_id; use crate::backend::vaapi::decoder::PoolCreationMode; use crate::backend::vaapi::decoder::VaStreamInfo; use crate::backend::vaapi::decoder::VaapiBackend; +use crate::backend::vaapi::decoder::VaapiPicture; use crate::codec::vp9::parser::BitDepth; use crate::codec::vp9::parser::Header; use crate::codec::vp9::parser::Profile; @@ -26,8 +27,9 @@ use crate::codec::vp9::parser::NUM_REF_FRAMES; use crate::decoder::stateless::vp9::Segmentation; use crate::decoder::stateless::vp9::StatelessVp9DecoderBackend; use crate::decoder::stateless::vp9::Vp9; +use crate::decoder::stateless::NewPictureError; +use crate::decoder::stateless::NewPictureResult; use crate::decoder::stateless::NewStatelessDecoderError; -use crate::decoder::stateless::StatelessBackendError; use crate::decoder::stateless::StatelessBackendResult; use crate::decoder::stateless::StatelessDecoder; use crate::decoder::stateless::StatelessDecoderBackendPicture; @@ -237,7 +239,7 @@ fn build_slice_param( } impl StatelessDecoderBackendPicture for VaapiBackend { - type Picture = (); + type Picture = VaapiPicture; } impl StatelessVp9DecoderBackend for VaapiBackend { @@ -245,14 +247,28 @@ impl StatelessVp9DecoderBackend for VaapiB self.new_sequence(header, PoolCreationMode::Highest) } + fn new_picture(&mut self, timestamp: u64) -> NewPictureResult { + let highest_pool = self.highest_pool(); + let surface = highest_pool + .get_surface() + .ok_or(NewPictureError::OutOfOutputBuffers)?; + let metadata = self.metadata_state.get_parsed()?; + let context = &metadata.context; + + Ok(VaPicture::new(timestamp, Rc::clone(context), surface)) + } + fn submit_picture( &mut self, - picture: &Header, + mut picture: Self::Picture, + hdr: &Header, reference_frames: &[Option; NUM_REF_FRAMES], bitstream: &[u8], - timestamp: u64, segmentation: &[Segmentation; MAX_SEGMENTS], ) -> StatelessBackendResult { + let metadata = self.metadata_state.get_parsed()?; + let context = &metadata.context; + let reference_frames: [u32; NUM_REF_FRAMES] = reference_frames .iter() .map(va_surface_id) @@ -260,16 +276,8 @@ impl StatelessVp9DecoderBackend for VaapiB .try_into() .unwrap(); - let highest_pool = self.highest_pool(); - let surface = highest_pool - .get_surface() - .ok_or(StatelessBackendError::OutOfResources)?; - - let metadata = self.metadata_state.get_parsed()?; - let context = &metadata.context; - let pic_param = context - .create_buffer(build_pic_param(picture, reference_frames)?) + .create_buffer(build_pic_param(hdr, reference_frames)?) .context("while creating pic params buffer")?; let slice_param = context @@ -280,14 +288,12 @@ impl StatelessVp9DecoderBackend for VaapiB .create_buffer(libva::BufferType::SliceData(Vec::from(bitstream))) .context("while creating slice data buffer")?; - let mut va_picture = VaPicture::new(timestamp, Rc::clone(context), surface); - // Add buffers with the parsed data. - va_picture.add_buffer(pic_param); - va_picture.add_buffer(slice_param); - va_picture.add_buffer(slice_data); + picture.add_buffer(pic_param); + picture.add_buffer(slice_param); + picture.add_buffer(slice_data); - self.process_picture::(va_picture) + self.process_picture::(picture) } }