Retain bins from frame to frame.

This PR makes Bevy keep entities in bins from frame to frame if they
haven't changed. This reduces the time spent in `queue_material_meshes`
and related functions to near zero for static geometry. This patch uses
the same change tick technique that #17567 to detect when meshes have
changed in such a way as to require re-binning.

In order to quickly find the relevant bin for an entity when that entity
has changed, we introduce a new type of cache, the *bin key cache*. This
cache stores a mapping from main world entity ID to cached bin key, as
well as the tick of the most recent change to the entity. As we iterate
through the visible entities in `queue_material_meshes`, we check the
cache to see whether the entity needs to be re-binned. If it doesn't,
then we mark it as clean in the `valid_cached_entity_bin_keys` bitset.
At the end, all bin keys not marked as clean are removed from the bins.

This patch has a dramatic effect on the rendering performance of most
benchmarks, as it effectively eliminates `queue_material_meshes` from
the profile. Note, however, that it generally simultaneously regresses
`batch_and_prepare_binned_render_phase` by a bit (not by enough to
outweigh the win, however). I believe that's because, before this patch,
`queue_material_meshes` put the bins in the CPU cache for
`batch_and_prepare_binned_render_phase` to use, while with this patch,
`batch_and_prepare_binned_render_phase` must load the batches into the
CPU cache itself.

Author: pcwalton

crates/bevy_core_pipeline/src/core_2d/mod.rs

@@ -436,8 +436,9 @@ pub fn extract_core_2d_camera_phases(
         let retained_view_entity = RetainedViewEntity::new(main_entity.into(), None, 0);
 
         transparent_2d_phases.insert_or_clear(retained_view_entity);
-        opaque_2d_phases.insert_or_clear(retained_view_entity, GpuPreprocessingMode::None);
-        alpha_mask_2d_phases.insert_or_clear(retained_view_entity, GpuPreprocessingMode::None);
+        opaque_2d_phases.prepare_for_new_frame(retained_view_entity, GpuPreprocessingMode::None);
+        alpha_mask_2d_phases
+            .prepare_for_new_frame(retained_view_entity, GpuPreprocessingMode::None);
 
         live_entities.insert(retained_view_entity);
     }

crates/bevy_core_pipeline/src/core_3d/mod.rs

@@ -629,8 +629,8 @@ pub fn extract_core_3d_camera_phases(
         // This is the main 3D camera, so use the first subview index (0).
         let retained_view_entity = RetainedViewEntity::new(main_entity.into(), None, 0);
 
-        opaque_3d_phases.insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
-        alpha_mask_3d_phases.insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+        opaque_3d_phases.prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
+        alpha_mask_3d_phases.prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
         transmissive_3d_phases.insert_or_clear(retained_view_entity);
         transparent_3d_phases.insert_or_clear(retained_view_entity);
 
@@ -698,18 +698,20 @@ pub fn extract_camera_prepass_phase(
         let retained_view_entity = RetainedViewEntity::new(main_entity.into(), None, 0);
 
         if depth_prepass || normal_prepass || motion_vector_prepass {
-            opaque_3d_prepass_phases.insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+            opaque_3d_prepass_phases
+                .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
             alpha_mask_3d_prepass_phases
-                .insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+                .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
         } else {
             opaque_3d_prepass_phases.remove(&retained_view_entity);
             alpha_mask_3d_prepass_phases.remove(&retained_view_entity);
         }
 
         if deferred_prepass {
-            opaque_3d_deferred_phases.insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+            opaque_3d_deferred_phases
+                .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
             alpha_mask_3d_deferred_phases
-                .insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+                .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
         } else {
             opaque_3d_deferred_phases.remove(&retained_view_entity);
             alpha_mask_3d_deferred_phases.remove(&retained_view_entity);

crates/bevy_pbr/src/material.rs

@@ -940,12 +940,20 @@ pub fn queue_material_meshes<M: Material>(
 
         let rangefinder = view.rangefinder3d();
         for (render_entity, visible_entity) in visible_entities.iter::<Mesh3d>() {
-            let Some(pipeline_id) = specialized_material_pipeline_cache
+            let Some((current_change_tick, pipeline_id)) = specialized_material_pipeline_cache
                 .get(&(*view_entity, *visible_entity))
-                .map(|(_, pipeline_id)| *pipeline_id)
+                .map(|(current_change_tick, pipeline_id)| (*current_change_tick, *pipeline_id))
             else {
                 continue;
             };
+
+            // Skip the entity if it's cached in a bin and up to date.
+            if opaque_phase.validate_cached_entity(*visible_entity, current_change_tick)
+                || alpha_mask_phase.validate_cached_entity(*visible_entity, current_change_tick)
+            {
+                continue;
+            }
+
             let Some(material_asset_id) = render_material_instances.get(visible_entity) else {
                 continue;
             };
@@ -997,6 +1005,7 @@ pub fn queue_material_meshes<M: Material>(
                             mesh_instance.should_batch(),
                             &gpu_preprocessing_support,
                         ),
+                        current_change_tick,
                     );
                 }
                 // Alpha mask
@@ -1019,6 +1028,7 @@ pub fn queue_material_meshes<M: Material>(
                             mesh_instance.should_batch(),
                             &gpu_preprocessing_support,
                         ),
+                        current_change_tick,
                     );
                 }
                 RenderPhaseType::Transparent => {
@@ -1036,6 +1046,10 @@ pub fn queue_material_meshes<M: Material>(
                 }
             }
         }
+
+        // Remove invalid entities from the bins.
+        opaque_phase.sweep_old_entities();
+        alpha_mask_phase.sweep_old_entities();
     }
 }
 

crates/bevy_pbr/src/prepass/mod.rs

@@ -1089,11 +1089,21 @@ pub fn queue_prepass_material_meshes<M: Material>(
         }
 
         for (render_entity, visible_entity) in visible_entities.iter::<Mesh3d>() {
-            let Some((_, pipeline_id)) =
+            let Some((current_change_tick, pipeline_id)) =
                 specialized_material_pipeline_cache.get(&(*view_entity, *visible_entity))
             else {
                 continue;
             };
+
+            // Skip the entity if it's cached in a bin and up to date.
+            if opaque_phase.as_mut().is_some_and(|opaque_phase| {
+                opaque_phase.validate_cached_entity(*visible_entity, *current_change_tick)
+            }) || alpha_mask_phase.as_mut().is_some_and(|alpha_mask_phase| {
+                alpha_mask_phase.validate_cached_entity(*visible_entity, *current_change_tick)
+            }) {
+                continue;
+            }
+
             let Some(material_asset_id) = render_material_instances.get(visible_entity) else {
                 continue;
             };
@@ -1134,6 +1144,7 @@ pub fn queue_prepass_material_meshes<M: Material>(
                                 mesh_instance.should_batch(),
                                 &gpu_preprocessing_support,
                             ),
+                            *current_change_tick,
                         );
                     } else if let Some(opaque_phase) = opaque_phase.as_mut() {
                         let (vertex_slab, index_slab) =
@@ -1157,6 +1168,7 @@ pub fn queue_prepass_material_meshes<M: Material>(
                                 mesh_instance.should_batch(),
                                 &gpu_preprocessing_support,
                             ),
+                            *current_change_tick,
                         );
                     }
                 }
@@ -1182,6 +1194,7 @@ pub fn queue_prepass_material_meshes<M: Material>(
                                 mesh_instance.should_batch(),
                                 &gpu_preprocessing_support,
                             ),
+                            *current_change_tick,
                         );
                     } else if let Some(alpha_mask_phase) = alpha_mask_phase.as_mut() {
                         let (vertex_slab, index_slab) =
@@ -1204,12 +1217,21 @@ pub fn queue_prepass_material_meshes<M: Material>(
                                 mesh_instance.should_batch(),
                                 &gpu_preprocessing_support,
                             ),
+                            *current_change_tick,
                         );
                     }
                 }
                 _ => {}
             }
         }
+
+        // Remove invalid entities from the bins.
+        if let Some(opaque_phase) = opaque_phase {
+            opaque_phase.sweep_old_entities();
+        }
+        if let Some(alpha_mask_phase) = alpha_mask_phase {
+            alpha_mask_phase.sweep_old_entities();
+        }
     }
 }
 

crates/bevy_pbr/src/render/light.rs

@@ -1299,7 +1299,7 @@ pub fn prepare_lights(
                 if first {
                     // Subsequent views with the same light entity will reuse the same shadow map
                     shadow_render_phases
-                        .insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+                        .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
                     live_shadow_mapping_lights.insert(retained_view_entity);
                 }
             }
@@ -1396,7 +1396,8 @@ pub fn prepare_lights(
 
             if first {
                 // Subsequent views with the same light entity will reuse the same shadow map
-                shadow_render_phases.insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+                shadow_render_phases
+                    .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
                 live_shadow_mapping_lights.insert(retained_view_entity);
             }
         }
@@ -1539,7 +1540,8 @@ pub fn prepare_lights(
                 // Subsequent views with the same light entity will **NOT** reuse the same shadow map
                 // (Because the cascades are unique to each view)
                 // TODO: Implement GPU culling for shadow passes.
-                shadow_render_phases.insert_or_clear(retained_view_entity, gpu_preprocessing_mode);
+                shadow_render_phases
+                    .prepare_for_new_frame(retained_view_entity, gpu_preprocessing_mode);
                 live_shadow_mapping_lights.insert(retained_view_entity);
             }
         }
@@ -1884,11 +1886,17 @@ pub fn queue_shadows<M: Material>(
             };
 
             for (entity, main_entity) in visible_entities.iter().copied() {
-                let Some((_, pipeline_id)) =
+                let Some((current_change_tick, pipeline_id)) =
                     specialized_material_pipeline_cache.get(&(view_light_entity, main_entity))
                 else {
                     continue;
                 };
+
+                // Skip the entity if it's cached in a bin and up to date.
+                if shadow_phase.validate_cached_entity(main_entity, *current_change_tick) {
+                    continue;
+                }
+
                 let Some(mesh_instance) = render_mesh_instances.render_mesh_queue_data(main_entity)
                 else {
                     continue;
@@ -1920,8 +1928,12 @@ pub fn queue_shadows<M: Material>(
                         mesh_instance.should_batch(),
                         &gpu_preprocessing_support,
                     ),
+                    *current_change_tick,
                 );
             }
+
+            // Remove invalid entities from the bins.
+            shadow_phase.sweep_old_entities();
         }
     }
 }

crates/bevy_render/Cargo.toml

@@ -96,13 +96,18 @@ smallvec = { version = "1.11", features = ["const_new"] }
 offset-allocator = "0.2"
 variadics_please = "1.1"
 tracing = { version = "0.1", default-features = false, features = ["std"] }
+indexmap = { version = "2" }
+fixedbitset = { version = "0.5" }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 # Omit the `glsl` feature in non-WebAssembly by default.
 naga_oil = { version = "0.16", default-features = false, features = [
   "test_shader",
 ] }
 
+[dev-dependencies]
+proptest = "1"
+
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 naga_oil = "0.16"
 js-sys = "0.3"

crates/bevy_render/src/batching/gpu_preprocessing.rs

@@ -4,6 +4,7 @@ use core::any::TypeId;
 
 use bevy_app::{App, Plugin};
 use bevy_ecs::{
+    prelude::Entity,
     query::{Has, With},
     resource::Resource,
     schedule::IntoSystemConfigs as _,
@@ -1326,8 +1327,9 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
                 let first_output_index = data_buffer.len() as u32;
                 let mut batch: Option<BinnedRenderPhaseBatch> = None;
 
-                for &(entity, main_entity) in &bin.entities {
-                    let Some(input_index) = GFBD::get_binned_index(&system_param_item, main_entity)
+                for main_entity in bin.entities() {
+                    let Some(input_index) =
+                        GFBD::get_binned_index(&system_param_item, *main_entity)
                     else {
                         continue;
                     };
@@ -1378,7 +1380,7 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
                                 },
                             );
                             batch = Some(BinnedRenderPhaseBatch {
-                                representative_entity: (entity, main_entity),
+                                representative_entity: (Entity::PLACEHOLDER, *main_entity),
                                 instance_range: output_index..output_index + 1,
                                 extra_index: PhaseItemExtraIndex::maybe_indirect_parameters_index(
                                     NonMaxU32::new(indirect_parameters_index),
@@ -1424,8 +1426,8 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
             let first_output_index = data_buffer.len() as u32;
 
             let mut batch: Option<BinnedRenderPhaseBatch> = None;
-            for &(entity, main_entity) in &phase.batchable_mesh_values[key].entities {
-                let Some(input_index) = GFBD::get_binned_index(&system_param_item, main_entity)
+            for main_entity in phase.batchable_mesh_values[key].entities() {
+                let Some(input_index) = GFBD::get_binned_index(&system_param_item, *main_entity)
                 else {
                     continue;
                 };
@@ -1487,7 +1489,7 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
                             },
                         );
                         batch = Some(BinnedRenderPhaseBatch {
-                            representative_entity: (entity, main_entity),
+                            representative_entity: (Entity::PLACEHOLDER, *main_entity),
                             instance_range: output_index..output_index + 1,
                             extra_index: PhaseItemExtraIndex::IndirectParametersIndex {
                                 range: indirect_parameters_index..(indirect_parameters_index + 1),
@@ -1507,7 +1509,7 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
                             },
                         );
                         batch = Some(BinnedRenderPhaseBatch {
-                            representative_entity: (entity, main_entity),
+                            representative_entity: (Entity::PLACEHOLDER, *main_entity),
                             instance_range: output_index..output_index + 1,
                             extra_index: PhaseItemExtraIndex::None,
                         });
@@ -1559,8 +1561,8 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
                 )
             };
 
-            for &(_, main_entity) in &unbatchables.entities {
-                let Some(input_index) = GFBD::get_binned_index(&system_param_item, main_entity)
+            for main_entity in unbatchables.entities.keys() {
+                let Some(input_index) = GFBD::get_binned_index(&system_param_item, *main_entity)
                 else {
                     continue;
                 };

crates/bevy_render/src/batching/mod.rs

@@ -182,8 +182,22 @@ where
     BPI: BinnedPhaseItem,
 {
     for phase in phases.values_mut() {
+        phase.multidrawable_mesh_keys.clear();
+        phase
+            .multidrawable_mesh_keys
+            .extend(phase.multidrawable_mesh_values.keys().cloned());
         phase.multidrawable_mesh_keys.sort_unstable();
+
+        phase.batchable_mesh_keys.clear();
+        phase
+            .batchable_mesh_keys
+            .extend(phase.batchable_mesh_values.keys().cloned());
         phase.batchable_mesh_keys.sort_unstable();
+
+        phase.unbatchable_mesh_keys.clear();
+        phase
+            .unbatchable_mesh_keys
+            .extend(phase.unbatchable_mesh_values.keys().cloned());
         phase.unbatchable_mesh_keys.sort_unstable();
     }
 }

crates/bevy_render/src/batching/no_gpu_preprocessing.rs

@@ -1,6 +1,7 @@
 //! Batching functionality when GPU preprocessing isn't in use.
 
 use bevy_derive::{Deref, DerefMut};
+use bevy_ecs::entity::Entity;
 use bevy_ecs::resource::Resource;
 use bevy_ecs::system::{Res, ResMut, StaticSystemParam};
 use smallvec::{smallvec, SmallVec};
@@ -109,9 +110,9 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
 
         for key in &phase.batchable_mesh_keys {
             let mut batch_set: SmallVec<[BinnedRenderPhaseBatch; 1]> = smallvec![];
-            for &(entity, main_entity) in &phase.batchable_mesh_values[key].entities {
+            for main_entity in phase.batchable_mesh_values[key].entities() {
                 let Some(buffer_data) =
-                    GFBD::get_binned_batch_data(&system_param_item, main_entity)
+                    GFBD::get_binned_batch_data(&system_param_item, *main_entity)
                 else {
                     continue;
                 };
@@ -128,7 +129,7 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
                             == PhaseItemExtraIndex::maybe_dynamic_offset(instance.dynamic_offset)
                 }) {
                     batch_set.push(BinnedRenderPhaseBatch {
-                        representative_entity: (entity, main_entity),
+                        representative_entity: (Entity::PLACEHOLDER, *main_entity),
                         instance_range: instance.index..instance.index,
                         extra_index: PhaseItemExtraIndex::maybe_dynamic_offset(
                             instance.dynamic_offset,
@@ -157,9 +158,9 @@ pub fn batch_and_prepare_binned_render_phase<BPI, GFBD>(
         // Prepare unbatchables.
         for key in &phase.unbatchable_mesh_keys {
             let unbatchables = phase.unbatchable_mesh_values.get_mut(key).unwrap();
-            for &(_, main_entity) in &unbatchables.entities {
+            for main_entity in unbatchables.entities.keys() {
                 let Some(buffer_data) =
-                    GFBD::get_binned_batch_data(&system_param_item, main_entity)
+                    GFBD::get_binned_batch_data(&system_param_item, *main_entity)
                 else {
                     continue;
                 };