decoder/stateless: let the backends decide which resources they need

src/codec/h265/picture.rs

@@ -53,7 +53,6 @@ impl PictureData {
         first_picture_after_eos: bool,
         prev_tid0_pic: Option<&PictureData>,
         max_pic_order_cnt_lsb: i32,
-        _timestamp: u64,
     ) -> Self {
         let hdr = &slice.header;
         let nalu_type = slice.nalu.header.type_;

src/decoder/stateless.rs

@@ -45,6 +45,8 @@ use crate::Resolution;
 pub enum StatelessBackendError {
     #[error("not enough resources to proceed with the operation now")]
     OutOfResources,
+    #[error("no frame pool can satisfy the requested frame resolution {0:?}")]
+    NoFramePool(Resolution),
     #[error(transparent)]
     Other(#[from] anyhow::Error),
 }

src/decoder/stateless/av1/dummy.rs

@@ -28,15 +28,23 @@ impl StatelessAV1DecoderBackend for Backend {
 
     fn new_picture(
         &mut self,
-        _: &crate::codec::av1::parser::SequenceHeaderObu,
         _: &crate::codec::av1::parser::FrameHeaderObu,
         _: u64,
-        _: &[Option<Self::Handle>; crate::codec::av1::parser::NUM_REF_FRAMES],
         _: Option<u32>,
     ) -> crate::decoder::stateless::StatelessBackendResult<Self::Picture> {
         Ok(())
     }
 
+    fn begin_picture(
+        &mut self,
+        _: &mut Self::Picture,
+        _: &crate::codec::av1::parser::SequenceHeaderObu,
+        _: &crate::codec::av1::parser::FrameHeaderObu,
+        _: &[Option<Self::Handle>; crate::codec::av1::parser::NUM_REF_FRAMES],
+    ) -> crate::decoder::stateless::StatelessBackendResult<()> {
+        Ok(())
+    }
+
     fn decode_tile_group(
         &mut self,
         _: &mut Self::Picture,

src/decoder/stateless/av1/vaapi.rs

@@ -31,6 +31,7 @@ use crate::decoder::stateless::av1::Av1;
 use crate::decoder::stateless::av1::StatelessAV1DecoderBackend;
 use crate::decoder::stateless::NewStatelessDecoderError;
 use crate::decoder::stateless::StatelessBackendError;
+use crate::decoder::stateless::StatelessBackendResult;
 use crate::decoder::stateless::StatelessDecoder;
 use crate::decoder::stateless::StatelessDecoderBackendPicture;
 use crate::decoder::BlockingMode;
@@ -507,7 +508,7 @@ impl<M: SurfaceMemoryDescriptor + 'static> StatelessAV1DecoderBackend for VaapiB
         &mut self,
         sequence: &Rc<SequenceHeaderObu>,
         highest_spatial_layer: Option<u32>,
-    ) -> crate::decoder::stateless::StatelessBackendResult<()> {
+    ) -> StatelessBackendResult<()> {
         let pool_creation_mode = match highest_spatial_layer {
             Some(highest_layer) => {
                 /* The spec mandates a 2:1 or 1.5:1 ratio, let's go with 2:1 to
@@ -526,50 +527,53 @@ impl<M: SurfaceMemoryDescriptor + 'static> StatelessAV1DecoderBackend for VaapiB
 
     fn new_picture(
         &mut self,
-        sequence: &SequenceHeaderObu,
         hdr: &FrameHeaderObu,
         timestamp: u64,
-        reference_frames: &[Option<Self::Handle>; NUM_REF_FRAMES],
         highest_spatial_layer: Option<u32>,
-    ) -> crate::decoder::stateless::StatelessBackendResult<Self::Picture> {
-        let surface = match highest_spatial_layer {
+    ) -> StatelessBackendResult<Self::Picture> {
+        let pool = match highest_spatial_layer {
             Some(_) => {
                 let layer = Resolution {
                     width: hdr.upscaled_width,
                     height: hdr.frame_height,
                 };
 
-                let pool = self
-                    .pool(layer)
-                    .ok_or(StatelessBackendError::Other(anyhow!(
-                        "No pool available for this layer"
-                    )))?;
-
-                pool.get_surface()
-                    .ok_or(StatelessBackendError::OutOfResources)?
-            }
-            None => {
-                let highest_pool = self.highest_pool();
-                highest_pool
-                    .get_surface()
-                    .ok_or(StatelessBackendError::OutOfResources)?
+                self.pool(layer)
+                    .ok_or(StatelessBackendError::NoFramePool(layer))?
             }
+            None => self.highest_pool(),
         };
 
+        let surface = pool
+            .get_surface()
+            .ok_or(StatelessBackendError::OutOfResources)?;
+
         let metadata = self.metadata_state.get_parsed()?;
-        let mut picture = VaPicture::new(timestamp, Rc::clone(&metadata.context), surface);
+        Ok(VaPicture::new(
+            timestamp,
+            Rc::clone(&metadata.context),
+            surface,
+        ))
+    }
 
-        let surface_id = picture.surface().id();
+    fn begin_picture(
+        &mut self,
+        picture: &mut Self::Picture,
+        sequence: &SequenceHeaderObu,
+        hdr: &FrameHeaderObu,
+        reference_frames: &[Option<Self::Handle>; NUM_REF_FRAMES],
+    ) -> StatelessBackendResult<()> {
+        let metadata = self.metadata_state.get_parsed()?;
 
-        let pic_param = build_pic_param(hdr, sequence, surface_id, reference_frames)
+        let pic_param = build_pic_param(hdr, sequence, picture.surface().id(), reference_frames)
             .context("Failed to build picture parameter")?;
         let pic_param = metadata
             .context
             .create_buffer(pic_param)
             .context("Failed to create picture parameter buffer")?;
         picture.add_buffer(pic_param);
 
-        Ok(picture)
+        Ok(())
     }
 
     fn decode_tile_group(
@@ -598,10 +602,7 @@ impl<M: SurfaceMemoryDescriptor + 'static> StatelessAV1DecoderBackend for VaapiB
         Ok(())
     }
 
-    fn submit_picture(
-        &mut self,
-        picture: Self::Picture,
-    ) -> crate::decoder::stateless::StatelessBackendResult<Self::Handle> {
+    fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle> {
         self.process_picture::<Av1>(picture)
     }
 }

src/decoder/stateless/h264.rs

@@ -41,6 +41,7 @@ use crate::codec::h264::picture::Reference;
 use crate::decoder::stateless::DecodeError;
 use crate::decoder::stateless::DecodingState;
 use crate::decoder::stateless::PoolLayer;
+use crate::decoder::stateless::StatelessBackendError;
 use crate::decoder::stateless::StatelessBackendResult;
 use crate::decoder::stateless::StatelessCodec;
 use crate::decoder::stateless::StatelessDecoder;
@@ -50,7 +51,6 @@ use crate::decoder::stateless::TryFormat;
 use crate::decoder::BlockingMode;
 use crate::decoder::DecodedHandle;
 use crate::decoder::DecoderEvent;
-use crate::decoder::FramePool;
 use crate::decoder::StreamInfo;
 use crate::Resolution;
 
@@ -84,19 +84,14 @@ pub trait StatelessH264DecoderBackend:
     fn new_sequence(&mut self, sps: &Rc<Sps>) -> StatelessBackendResult<()>;
 
     /// Called when the decoder determines that a frame or field was found.
-    fn new_picture(
-        &mut self,
-        picture: &PictureData,
-        timestamp: u64,
-    ) -> StatelessBackendResult<Self::Picture>;
+    fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult<Self::Picture>;
 
     /// Called when the decoder determines that a second field was found.
     /// Indicates that the underlying BackendHandle is to be shared between the
     /// two pictures. This is so both fields decode to the same underlying
     /// resource and can thus be presented together as a single frame.
     fn new_field_picture(
         &mut self,
-        picture: &PictureData,
         timestamp: u64,
         first_field: &Self::Handle,
     ) -> StatelessBackendResult<Self::Picture>;
@@ -1098,6 +1093,18 @@ where
         timestamp: u64,
         slice: &Slice,
     ) -> Result<CurrentPicState<B::Picture>, DecodeError> {
+        // Start by securing the backend picture before modifying our state.
+        let first_field = self.codec.find_first_field(&slice.header)?;
+        let mut backend_pic = if let Some(first_field) = &first_field {
+            self.backend.new_field_picture(timestamp, &first_field.1)
+        } else {
+            self.backend.new_picture(timestamp)
+        }
+        .map_err(|e| match e {
+            StatelessBackendError::OutOfResources => DecodeError::NotEnoughOutputBuffers(1),
+            e => DecodeError::BackendError(e),
+        })?;
+
         let nalu_hdr = &slice.nalu.header;
 
         if nalu_hdr.idr_pic_flag {
@@ -1120,17 +1127,6 @@ where
             return Err(DecodeError::CheckEvents);
         }
 
-        if self
-            .backend
-            .frame_pool(PoolLayer::Highest)
-            .pop()
-            .ok_or(anyhow!("No pool found"))?
-            .num_free_frames()
-            == 0
-        {
-            return Err(DecodeError::NotEnoughOutputBuffers(1));
-        }
-
         let current_macroblock = match pps.sps.separate_colour_plane_flag {
             true => CurrentMacroblockTracking::SeparateColorPlane(Default::default()),
             false => CurrentMacroblockTracking::NonSeparateColorPlane(0),
@@ -1142,7 +1138,6 @@ where
             self.handle_frame_num_gap(&pps.sps, frame_num, timestamp)?;
         }
 
-        let first_field = self.codec.find_first_field(&slice.header)?;
         let pic = self.init_current_pic(
             slice,
             &pps.sps,
@@ -1153,13 +1148,6 @@ where
 
         debug!("Decode picture POC {:?}", pic.pic_order_cnt);
 
-        let mut backend_pic = if let Some(first_field) = first_field {
-            self.backend
-                .new_field_picture(&pic, timestamp, &first_field.1)?
-        } else {
-            self.backend.new_picture(&pic, timestamp)?
-        };
-
         self.backend.start_picture(
             &mut backend_pic,
             &pic,

src/decoder/stateless/h264/dummy.rs

@@ -41,12 +41,7 @@ impl StatelessH264DecoderBackend for Backend {
         Ok(())
     }
 
-    fn new_field_picture(
-        &mut self,
-        _: &PictureData,
-        _: u64,
-        _: &Self::Handle,
-    ) -> StatelessBackendResult<()> {
+    fn new_field_picture(&mut self, _: u64, _: &Self::Handle) -> StatelessBackendResult<()> {
         Ok(())
     }
 
@@ -68,7 +63,7 @@ impl StatelessH264DecoderBackend for Backend {
         })
     }
 
-    fn new_picture(&mut self, _: &PictureData, _: u64) -> StatelessBackendResult<()> {
+    fn new_picture(&mut self, _: u64) -> StatelessBackendResult<()> {
         Ok(())
     }
 }

src/decoder/stateless/h264/vaapi.rs

@@ -540,11 +540,7 @@ impl<M: SurfaceMemoryDescriptor + 'static> StatelessH264DecoderBackend for Vaapi
         self.process_picture::<H264>(picture)
     }
 
-    fn new_picture(
-        &mut self,
-        _: &PictureData,
-        timestamp: u64,
-    ) -> StatelessBackendResult<Self::Picture> {
+    fn new_picture(&mut self, timestamp: u64) -> StatelessBackendResult<Self::Picture> {
         let highest_pool = self.highest_pool();
         let surface = highest_pool
             .get_surface()
@@ -561,7 +557,6 @@ impl<M: SurfaceMemoryDescriptor + 'static> StatelessH264DecoderBackend for Vaapi
 
     fn new_field_picture(
         &mut self,
-        _: &PictureData,
         timestamp: u64,
         first_field: &Self::Handle,
     ) -> StatelessBackendResult<Self::Picture> {