From 96a8192722bef2952a749b8e5ea5980fbe67f360 Mon Sep 17 00:00:00 2001 From: james7132 Date: Fri, 28 Oct 2022 04:56:13 -0700 Subject: [PATCH 1/6] Unionize the Fetches and the Queries --- crates/bevy_ecs/src/query/fetch.rs | 207 +++++++++++++++++++--------- crates/bevy_ecs/src/query/filter.rs | 35 +++-- crates/bevy_ecs/src/query/iter.rs | 124 +++++++++++++---- 3 files changed, 261 insertions(+), 105 deletions(-) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index 52886817e20e0..eb9eb2be948aa 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -10,7 +10,7 @@ use crate::{ use bevy_ecs_macros::all_tuples; pub use bevy_ecs_macros::WorldQuery; use bevy_ptr::{ThinSlicePtr, UnsafeCellDeref}; -use std::{cell::UnsafeCell, marker::PhantomData}; +use std::{cell::UnsafeCell, marker::PhantomData, mem::ManuallyDrop}; /// Types that can be fetched from a [`World`] using a [`Query`]. /// @@ -528,11 +528,14 @@ impl<'w> WorldQueryGats<'w> for Entity { unsafe impl ReadOnlyWorldQuery for Entity {} #[doc(hidden)] -pub struct ReadFetch<'w, T> { - // T::Storage = TableStorage - table_components: Option>>, - // T::Storage = SparseStorage - sparse_set: Option<&'w ComponentSparseSet>, +pub struct ReadFetch<'w, T: Component> { + components: StorageSwitch< + T, + // T::Storage = TableStorage + Option>>, + // T::Storage = SparseStorage + &'w ComponentSparseSet, + >, } /// SAFETY: `ROQueryFetch` is the same as `QueryFetch` @@ -560,14 +563,16 @@ unsafe impl WorldQuery for &T { _change_tick: u32, ) -> ReadFetch<'w, T> { ReadFetch { - table_components: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { - world - .storages() - .sparse_sets - .get(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) - }), + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( + world + .storages() + .sparse_sets + .get(component_id) + .unwrap_or_else(|| debug_checked_unreachable()), + ), + }, } } @@ -575,8 +580,7 @@ unsafe impl WorldQuery for &T { fetch: &>::Fetch, ) -> >::Fetch { ReadFetch { - table_components: fetch.table_components, - sparse_set: fetch.sparse_set, + components: fetch.components, } } @@ -598,13 +602,13 @@ unsafe impl WorldQuery for &T { &component_id: &ComponentId, table: &'w Table, ) { - fetch.table_components = Some( + fetch.components = StorageSwitch::new_table(Some( table .get_column(component_id) .unwrap_or_else(|| debug_checked_unreachable()) .get_data_slice() .into(), - ); + )); } #[inline(always)] @@ -615,13 +619,14 @@ unsafe impl WorldQuery for &T { ) -> >::Item { match T::Storage::STORAGE_TYPE { StorageType::Table => fetch - .table_components + .components + .table() .unwrap_or_else(|| debug_checked_unreachable()) .get(table_row) .deref(), StorageType::SparseSet => fetch - .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .components + .sparse_set() .get(entity) .unwrap_or_else(|| debug_checked_unreachable()) .deref(), @@ -671,15 +676,17 @@ impl<'w, T: Component> WorldQueryGats<'w> for &T { } #[doc(hidden)] -pub struct WriteFetch<'w, T> { - // T::Storage = TableStorage - table_data: Option<( - ThinSlicePtr<'w, UnsafeCell>, - ThinSlicePtr<'w, UnsafeCell>, - )>, - // T::Storage = SparseStorage - sparse_set: Option<&'w ComponentSparseSet>, - +pub struct WriteFetch<'w, T: Component> { + components: StorageSwitch< + T, + // T::Storage = TableStorage + Option<( + ThinSlicePtr<'w, UnsafeCell>, + ThinSlicePtr<'w, UnsafeCell>, + )>, + // T::Storage = SparseStorage + &'w ComponentSparseSet, + >, last_change_tick: u32, change_tick: u32, } @@ -709,14 +716,16 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { change_tick: u32, ) -> WriteFetch<'w, T> { WriteFetch { - table_data: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { - world - .storages() - .sparse_sets - .get(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) - }), + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( + world + .storages() + .sparse_sets + .get(component_id) + .unwrap_or_else(|| debug_checked_unreachable()), + ), + }, last_change_tick, change_tick, } @@ -726,8 +735,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { fetch: &>::Fetch, ) -> >::Fetch { WriteFetch { - table_data: fetch.table_data, - sparse_set: fetch.sparse_set, + components: fetch.components, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, } @@ -754,10 +762,10 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { let column = table .get_column(component_id) .unwrap_or_else(|| debug_checked_unreachable()); - fetch.table_data = Some(( + fetch.components = StorageSwitch::new_table(Some(( column.get_data_slice().into(), column.get_ticks_slice().into(), - )); + ))); } #[inline(always)] @@ -769,7 +777,8 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { match T::Storage::STORAGE_TYPE { StorageType::Table => { let (table_components, table_ticks) = fetch - .table_data + .components + .table() .unwrap_or_else(|| debug_checked_unreachable()); Mut { value: table_components.get(table_row).deref_mut(), @@ -782,8 +791,8 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { } StorageType::SparseSet => { let (component, component_ticks) = fetch - .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .components + .sparse_set() .get_with_ticks(entity) .unwrap_or_else(|| debug_checked_unreachable()); Mut { @@ -1025,12 +1034,14 @@ impl ChangeTrackers { } #[doc(hidden)] -pub struct ChangeTrackersFetch<'w, T> { - // T::Storage = TableStorage - table_ticks: Option>>, - // T::Storage = SparseStorage - sparse_set: Option<&'w ComponentSparseSet>, - +pub struct ChangeTrackersFetch<'w, T: Component> { + components: StorageSwitch< + T, + // T::Storage = TableStorage + Option>>, + // T::Storage = SparseStorage + &'w ComponentSparseSet, + >, marker: PhantomData, last_change_tick: u32, change_tick: u32, @@ -1061,14 +1072,16 @@ unsafe impl WorldQuery for ChangeTrackers { change_tick: u32, ) -> ChangeTrackersFetch<'w, T> { ChangeTrackersFetch { - table_ticks: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { - world - .storages() - .sparse_sets - .get(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) - }), + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( + world + .storages() + .sparse_sets + .get(component_id) + .unwrap_or_else(|| debug_checked_unreachable()), + ), + }, marker: PhantomData, last_change_tick, change_tick, @@ -1079,8 +1092,7 @@ unsafe impl WorldQuery for ChangeTrackers { fetch: &>::Fetch, ) -> >::Fetch { ChangeTrackersFetch { - table_ticks: fetch.table_ticks, - sparse_set: fetch.sparse_set, + components: fetch.components, marker: fetch.marker, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, @@ -1105,13 +1117,13 @@ unsafe impl WorldQuery for ChangeTrackers { &id: &ComponentId, table: &'w Table, ) { - fetch.table_ticks = Some( + fetch.components = StorageSwitch::new_table(Some( table .get_column(id) .unwrap_or_else(|| debug_checked_unreachable()) .get_ticks_slice() .into(), - ); + )); } #[inline(always)] @@ -1124,7 +1136,8 @@ unsafe impl WorldQuery for ChangeTrackers { StorageType::Table => ChangeTrackers { component_ticks: { let table_ticks = fetch - .table_ticks + .components + .table() .unwrap_or_else(|| debug_checked_unreachable()); table_ticks.get(table_row).read() }, @@ -1134,8 +1147,8 @@ unsafe impl WorldQuery for ChangeTrackers { }, StorageType::SparseSet => ChangeTrackers { component_ticks: *fetch - .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .components + .sparse_set() .get_ticks(entity) .unwrap_or_else(|| debug_checked_unreachable()) .get(), @@ -1519,3 +1532,65 @@ impl<'a, Q: WorldQuery> WorldQueryGats<'a> for NopWorldQuery { } /// SAFETY: `NopFetch` never accesses any data unsafe impl ReadOnlyWorldQuery for NopWorldQuery {} + +// A compile-time checked union of two different types that differs based +// on the storage type of a given Component. +pub(super) union StorageSwitch { + table: ManuallyDrop, + sparse_set: ManuallyDrop, + marker: PhantomData, +} + +impl StorageSwitch { + pub const fn new_table(table: A) -> Self { + Self { + table: ManuallyDrop::new(table), + } + } + + pub const fn new_sparse_set(sparse_set: B) -> Self { + Self { + sparse_set: ManuallyDrop::new(sparse_set), + } + } +} + +impl StorageSwitch { + pub fn table(&self) -> A { + // SAFETY: The variant of the union is checked at compile time + unsafe { + match T::Storage::STORAGE_TYPE { + StorageType::Table => *self.table, + _ => debug_checked_unreachable(), + } + } + } + + pub fn sparse_set(&self) -> B { + // SAFETY: The variant of the union is checked at compile time + unsafe { + match T::Storage::STORAGE_TYPE { + StorageType::SparseSet => *self.sparse_set, + _ => debug_checked_unreachable(), + } + } + } +} + +impl Clone for StorageSwitch { + fn clone(&self) -> Self { + // SAFETY: The variant of the union is checked at compile time + unsafe { + match T::Storage::STORAGE_TYPE { + StorageType::Table => Self { + table: self.table.clone(), + }, + StorageType::SparseSet => Self { + sparse_set: self.sparse_set.clone(), + }, + } + } + } +} + +impl Copy for StorageSwitch {} diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index 1571907f567a6..8d89d5ee7e251 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -3,7 +3,8 @@ use crate::{ component::{Component, ComponentId, ComponentStorage, ComponentTicks, StorageType}, entity::Entity, query::{ - debug_checked_unreachable, Access, FilteredAccess, QueryFetch, WorldQuery, WorldQueryGats, + debug_checked_unreachable, fetch::StorageSwitch, Access, FilteredAccess, QueryFetch, + WorldQuery, WorldQueryGats, }, storage::{ComponentSparseSet, Table}, world::World, @@ -438,10 +439,13 @@ macro_rules! impl_tick_filter { #[doc(hidden)] $(#[$fetch_meta])* - pub struct $fetch_name<'w, T> { - table_ticks: Option>>, + pub struct $fetch_name<'w, T: Component> { + components: StorageSwitch< + T, + Option>>, + &'w ComponentSparseSet, + >, marker: PhantomData, - sparse_set: Option<&'w ComponentSparseSet>, last_change_tick: u32, change_tick: u32, } @@ -457,14 +461,15 @@ macro_rules! impl_tick_filter { unsafe fn init_fetch<'w>(world: &'w World, &id: &ComponentId, last_change_tick: u32, change_tick: u32) -> >::Fetch { QueryFetch::<'w, Self> { - table_ticks: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet) - .then(|| { + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( world.storages() .sparse_sets .get(id) .unwrap_or_else(|| debug_checked_unreachable()) - }), + ), + }, marker: PhantomData, last_change_tick, change_tick, @@ -475,8 +480,7 @@ macro_rules! impl_tick_filter { fetch: &>::Fetch, ) -> >::Fetch { $fetch_name { - table_ticks: fetch.table_ticks, - sparse_set: fetch.sparse_set, + components: fetch.components, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, marker: PhantomData, @@ -498,12 +502,12 @@ macro_rules! impl_tick_filter { &component_id: &ComponentId, table: &'w Table ) { - fetch.table_ticks = Some( + fetch.components = StorageSwitch::new_table(Some( table.get_column(component_id) .unwrap_or_else(|| debug_checked_unreachable()) .get_ticks_slice() .into() - ); + )); } #[inline] @@ -527,7 +531,8 @@ macro_rules! impl_tick_filter { match T::Storage::STORAGE_TYPE { StorageType::Table => { $is_detected(&*( - fetch.table_ticks + fetch.components + .table() .unwrap_or_else(|| debug_checked_unreachable()) .get(table_row)) .deref(), @@ -537,8 +542,8 @@ macro_rules! impl_tick_filter { } StorageType::SparseSet => { let ticks = &*fetch - .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .components + .sparse_set() .get_ticks(entity) .unwrap_or_else(|| debug_checked_unreachable()) .get(); diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index 8a693eabb02c4..c23ee7724753e 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -2,10 +2,16 @@ use crate::{ archetype::{ArchetypeEntity, ArchetypeId, Archetypes}, entity::{Entities, Entity}, prelude::World, - query::{ArchetypeFilter, QueryState, WorldQuery}, + ptr::ThinSlicePtr, + query::{debug_checked_unreachable, ArchetypeFilter, QueryState, WorldQuery}, storage::{TableId, Tables}, }; -use std::{borrow::Borrow, iter::FusedIterator, marker::PhantomData, mem::MaybeUninit}; +use std::{ + borrow::Borrow, + iter::FusedIterator, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, +}; use super::{QueryFetch, QueryItem, ReadOnlyWorldQuery}; @@ -464,10 +470,8 @@ impl<'w, 's, Q: ReadOnlyWorldQuery, F: ReadOnlyWorldQuery, const K: usize> Fused } struct QueryIterationCursor<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> { - table_id_iter: std::slice::Iter<'s, TableId>, - archetype_id_iter: std::slice::Iter<'s, ArchetypeId>, - table_entities: &'w [Entity], - archetype_entities: &'w [ArchetypeEntity], + id_iter: QuerySwitch, std::slice::Iter<'s, ArchetypeId>>, + entities: QuerySwitch, ThinSlicePtr<'w, ArchetypeEntity>>, fetch: QueryFetch<'w, Q>, filter: QueryFetch<'w, F>, // length of the table table or length of the archetype, depending on whether both `Q`'s and `F`'s fetches are dense @@ -486,10 +490,8 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, /// `archetype_index` or `table_row` to be alive at the same time. unsafe fn clone_cursor(&self) -> Self { Self { - table_id_iter: self.table_id_iter.clone(), - archetype_id_iter: self.archetype_id_iter.clone(), - table_entities: self.table_entities, - archetype_entities: self.archetype_entities, + id_iter: self.id_iter.clone(), + entities: self.entities.clone(), // SAFETY: upheld by caller invariants fetch: Q::clone_fetch(&self.fetch), filter: F::clone_fetch(&self.filter), @@ -510,8 +512,11 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, change_tick: u32, ) -> Self { QueryIterationCursor { - table_id_iter: [].iter(), - archetype_id_iter: [].iter(), + id_iter: if Self::IS_DENSE { + QuerySwitch::new_dense([].iter()) + } else { + QuerySwitch::new_sparse([].iter()) + }, ..Self::init(world, query_state, last_change_tick, change_tick) } } @@ -534,13 +539,21 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, last_change_tick, change_tick, ); + let table_entities: &[Entity] = &[]; + let archetype_entities: &[ArchetypeEntity] = &[]; QueryIterationCursor { fetch, filter, - table_entities: &[], - archetype_entities: &[], - table_id_iter: query_state.matched_table_ids.iter(), - archetype_id_iter: query_state.matched_archetype_ids.iter(), + id_iter: if Self::IS_DENSE { + QuerySwitch::new_dense(query_state.matched_table_ids.iter()) + } else { + QuerySwitch::new_sparse(query_state.matched_archetype_ids.iter()) + }, + entities: if Self::IS_DENSE { + QuerySwitch::new_dense(table_entities.into()) + } else { + QuerySwitch::new_sparse(archetype_entities.into()) + }, current_len: 0, current_index: 0, phantom: PhantomData, @@ -553,10 +566,10 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, if self.current_index > 0 { let index = self.current_index - 1; if Self::IS_DENSE { - let entity = self.table_entities.get_unchecked(index); + let entity = self.entities.dense().get(index); Some(Q::fetch(&mut self.fetch, *entity, index)) } else { - let archetype_entity = self.archetype_entities.get_unchecked(index); + let archetype_entity = self.entities.sparse().get(index); Some(Q::fetch( &mut self.fetch, archetype_entity.entity, @@ -585,13 +598,13 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, loop { // we are on the beginning of the query, or finished processing a table, so skip to the next if self.current_index == self.current_len { - let table_id = self.table_id_iter.next()?; + let table_id = self.id_iter.dense().next()?; let table = &tables[*table_id]; // SAFETY: `table` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with Q::set_table(&mut self.fetch, &query_state.fetch_state, table); F::set_table(&mut self.filter, &query_state.filter_state, table); - self.table_entities = table.entities(); + self.entities = QuerySwitch::new_dense(table.entities().into()); self.current_len = table.entity_count(); self.current_index = 0; continue; @@ -599,7 +612,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // SAFETY: set_table was called prior. // `current_index` is a table row in range of the current table, because if it was not, then the if above would have been executed. - let entity = self.table_entities.get_unchecked(self.current_index); + let entity = self.entities.dense().get(self.current_index); if !F::filter_fetch(&mut self.filter, *entity, self.current_index) { self.current_index += 1; continue; @@ -615,7 +628,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, } else { loop { if self.current_index == self.current_len { - let archetype_id = self.archetype_id_iter.next()?; + let archetype_id = self.id_iter.sparse().next()?; let archetype = &archetypes[*archetype_id]; // SAFETY: `archetype` and `tables` are from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -627,7 +640,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, archetype, table, ); - self.archetype_entities = archetype.entities(); + self.entities = QuerySwitch::new_sparse(archetype.entities().into()); self.current_len = archetype.len(); self.current_index = 0; continue; @@ -635,7 +648,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // SAFETY: set_archetype was called prior. // `current_index` is an archetype index row in range of the current archetype, because if it was not, then the if above would have been executed. - let archetype_entity = self.archetype_entities.get_unchecked(self.current_index); + let archetype_entity = self.entities.sparse().get(self.current_index); if !F::filter_fetch( &mut self.filter, archetype_entity.entity, @@ -658,3 +671,66 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, } } } + +// A compile-time checked union of two different types that differs based +// whether a fetch is dense or not. +union QuerySwitch { + dense: ManuallyDrop, + sparse: ManuallyDrop, + marker: PhantomData<(Q, F)>, +} + +impl QuerySwitch { + const IS_DENSE: bool = Q::IS_DENSE && F::IS_DENSE; + + pub const fn new_dense(dense: A) -> Self { + Self { + dense: ManuallyDrop::new(dense), + } + } + + pub const fn new_sparse(sparse: B) -> Self { + Self { + sparse: ManuallyDrop::new(sparse), + } + } + + pub fn dense(&mut self) -> &mut A { + // SAFETY: The variant of the union is checked at compile time + unsafe { + if Self::IS_DENSE { + &mut self.dense + } else { + debug_checked_unreachable() + } + } + } + + pub fn sparse(&mut self) -> &mut B { + // SAFETY: The variant of the union is checked at compile time + unsafe { + if !Self::IS_DENSE { + &mut self.sparse + } else { + debug_checked_unreachable() + } + } + } +} + +impl Clone for QuerySwitch { + fn clone(&self) -> Self { + // SAFETY: The variant of the union is checked at compile time + unsafe { + if Self::IS_DENSE { + Self { + dense: self.dense.clone(), + } + } else { + Self { + sparse: self.sparse.clone(), + } + } + } + } +} From c57b743a5e22df5144d4b95268593a31d4e6c6c3 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 31 Oct 2022 15:52:58 -0700 Subject: [PATCH 2/6] Document StorageSwitch --- crates/bevy_ecs/src/query/fetch.rs | 77 +++++++++++++++++++++--------- crates/bevy_ecs/src/query/mod.rs | 2 +- 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index eb9eb2be948aa..f4aa82fa2252c 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -1533,8 +1533,8 @@ impl<'a, Q: WorldQuery> WorldQueryGats<'a> for NopWorldQuery { /// SAFETY: `NopFetch` never accesses any data unsafe impl ReadOnlyWorldQuery for NopWorldQuery {} -// A compile-time checked union of two different types that differs based -// on the storage type of a given Component. +/// A compile-time checked union of two different types that differs based +/// on the storage type of a given Component. pub(super) union StorageSwitch { table: ManuallyDrop, sparse_set: ManuallyDrop, @@ -1542,37 +1542,70 @@ pub(super) union StorageSwitch { } impl StorageSwitch { - pub const fn new_table(table: A) -> Self { - Self { - table: ManuallyDrop::new(table), + /// Creates a new [`StorageSwitch`] using a table variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a table component. + /// + /// # Safety + /// `T` must be a table component. + pub const unsafe fn new_table(table: A) -> Self { + match T::Storage::STORAGE_TYPE { + StorageType::Table => Self { + table: ManuallyDrop::new(table), + }, + _ => debug_checked_unreachable(), } } - pub const fn new_sparse_set(sparse_set: B) -> Self { - Self { - sparse_set: ManuallyDrop::new(sparse_set), + /// Creates a new [`StorageSwitch`] using a sparse set variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a sparse set component. + /// + /// # Safety + /// `T` must be a sparse set component. + pub const unsafe fn new_sparse_set(sparse_set: B) -> Self { + // SAFETY: The variant of the union is checked at compile time + match T::Storage::STORAGE_TYPE { + StorageType::Table => Self { + sparse_set: ManuallyDrop::new(sparse_set), + }, + _ => debug_checked_unreachable(), } } } impl StorageSwitch { - pub fn table(&self) -> A { - // SAFETY: The variant of the union is checked at compile time - unsafe { - match T::Storage::STORAGE_TYPE { - StorageType::Table => *self.table, - _ => debug_checked_unreachable(), - } + /// Fetches the internal value as a table variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a table component. + /// + /// # Safety + /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_table`]. + pub unsafe fn table(&self) -> A { + match T::Storage::STORAGE_TYPE { + StorageType::Table => *self.table, + // SAFETY: If T::Storage::StorageType is set to table, new_table + // would have panicked instead of producing a value. + _ => debug_checked_unreachable(), } } - pub fn sparse_set(&self) -> B { - // SAFETY: The variant of the union is checked at compile time - unsafe { - match T::Storage::STORAGE_TYPE { - StorageType::SparseSet => *self.sparse_set, - _ => debug_checked_unreachable(), - } + /// Fetches the internal value as a sparse set variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a sparse set component. + /// + /// # Safety + /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_sparse_set`]. + pub unsafe fn sparse_set(&self) -> B { + match T::Storage::STORAGE_TYPE { + StorageType::SparseSet => *self.sparse_set, + // SAFETY: If T::Storage::StorageType is set to table, new_sparse_set + // would have panicked instead of producing a value. + _ => debug_checked_unreachable(), } } } diff --git a/crates/bevy_ecs/src/query/mod.rs b/crates/bevy_ecs/src/query/mod.rs index dff4b9a251f41..169194b0e1e06 100644 --- a/crates/bevy_ecs/src/query/mod.rs +++ b/crates/bevy_ecs/src/query/mod.rs @@ -11,7 +11,7 @@ pub use iter::*; pub use state::*; #[allow(unreachable_code)] -pub(crate) unsafe fn debug_checked_unreachable() -> ! { +pub(crate) const unsafe fn debug_checked_unreachable() -> ! { #[cfg(debug_assertions)] unreachable!(); std::hint::unreachable_unchecked(); From ea6674478d4e5c2924e188aca2cb55ff8014c44f Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 31 Oct 2022 15:58:04 -0700 Subject: [PATCH 3/6] Document QuerySwitch --- crates/bevy_ecs/src/query/fetch.rs | 20 +++++----- crates/bevy_ecs/src/query/iter.rs | 60 +++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index f4aa82fa2252c..042b1830bb710 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -1543,10 +1543,10 @@ pub(super) union StorageSwitch { impl StorageSwitch { /// Creates a new [`StorageSwitch`] using a table variant. - /// + /// /// # Panics /// This will panic on debug builds if `T` is not a table component. - /// + /// /// # Safety /// `T` must be a table component. pub const unsafe fn new_table(table: A) -> Self { @@ -1559,10 +1559,10 @@ impl StorageSwitch { } /// Creates a new [`StorageSwitch`] using a sparse set variant. - /// + /// /// # Panics /// This will panic on debug builds if `T` is not a sparse set component. - /// + /// /// # Safety /// `T` must be a sparse set component. pub const unsafe fn new_sparse_set(sparse_set: B) -> Self { @@ -1578,11 +1578,11 @@ impl StorageSwitch { impl StorageSwitch { /// Fetches the internal value as a table variant. - /// + /// /// # Panics /// This will panic on debug builds if `T` is not a table component. - /// - /// # Safety + /// + /// # Safety /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_table`]. pub unsafe fn table(&self) -> A { match T::Storage::STORAGE_TYPE { @@ -1594,11 +1594,11 @@ impl StorageSwitch { } /// Fetches the internal value as a sparse set variant. - /// + /// /// # Panics /// This will panic on debug builds if `T` is not a sparse set component. - /// - /// # Safety + /// + /// # Safety /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_sparse_set`]. pub unsafe fn sparse_set(&self) -> B { match T::Storage::STORAGE_TYPE { diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index c23ee7724753e..9c5f70a968020 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -672,8 +672,8 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, } } -// A compile-time checked union of two different types that differs based -// whether a fetch is dense or not. +/// A compile-time checked union of two different types that differs based +/// whether a fetch is dense or not. union QuerySwitch { dense: ManuallyDrop, sparse: ManuallyDrop, @@ -683,19 +683,51 @@ union QuerySwitch { impl QuerySwitch { const IS_DENSE: bool = Q::IS_DENSE && F::IS_DENSE; - pub const fn new_dense(dense: A) -> Self { - Self { - dense: ManuallyDrop::new(dense), + /// Creates a new [`QuerySwitch`] of the dense variant. + /// + /// # Panics + /// Will panic in debug mode if either `Q::IS_DENSE` and `F::IS_DENSE` + /// are not true. + /// + /// # Safety + /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + pub const unsafe fn new_dense(dense: A) -> Self { + if Self::IS_DENSE { + Self { + dense: ManuallyDrop::new(dense), + } + } else { + debug_checked_unreachable() } } - pub const fn new_sparse(sparse: B) -> Self { - Self { - sparse: ManuallyDrop::new(sparse), + /// Creates a new [`QuerySwitch`] of the sparse variant. + /// + /// # Panics + /// Will panic in debug mode if both `Q::IS_DENSE` and `F::IS_DENSE` + /// are true. + /// + /// # Safety + /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + pub const unsafe fn new_sparse(sparse: B) -> Self { + if !Self::IS_DENSE { + Self { + sparse: ManuallyDrop::new(sparse), + } + } else { + debug_checked_unreachable() } } - pub fn dense(&mut self) -> &mut A { + /// Fetches a mutable reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if either `Q::IS_DENSE` and `F::IS_DENSE` + /// are not true. + /// + /// # Safety + /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + pub unsafe fn dense(&mut self) -> &mut A { // SAFETY: The variant of the union is checked at compile time unsafe { if Self::IS_DENSE { @@ -706,7 +738,15 @@ impl QuerySwitch { } } - pub fn sparse(&mut self) -> &mut B { + /// Fetches a mutable reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if both `Q::IS_DENSE` and `F::IS_DENSE` + /// are true. + /// + /// # Safety + /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + pub unsafe fn sparse(&mut self) -> &mut B { // SAFETY: The variant of the union is checked at compile time unsafe { if !Self::IS_DENSE { From ca0f9c4b108c3524e3807267bb7ce9b8d773d019 Mon Sep 17 00:00:00 2001 From: james7132 Date: Mon, 31 Oct 2022 17:04:40 -0700 Subject: [PATCH 4/6] Fix CI --- crates/bevy_ecs/src/query/fetch.rs | 4 ++-- crates/bevy_ecs/src/query/iter.rs | 26 ++++++++++++-------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index 042b1830bb710..a48d870ef28d5 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -1564,11 +1564,11 @@ impl StorageSwitch { /// This will panic on debug builds if `T` is not a sparse set component. /// /// # Safety - /// `T` must be a sparse set component. + /// `T` must be a component. pub const unsafe fn new_sparse_set(sparse_set: B) -> Self { // SAFETY: The variant of the union is checked at compile time match T::Storage::STORAGE_TYPE { - StorageType::Table => Self { + StorageType::SparseSet => Self { sparse_set: ManuallyDrop::new(sparse_set), }, _ => debug_checked_unreachable(), diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index 9c5f70a968020..d96eb98574de4 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -681,6 +681,10 @@ union QuerySwitch { } impl QuerySwitch { + /// Whether the corresponding query is using dense iteration or not. + /// For more information, see [`WorldQuery::IS_DENSE`] + /// + /// [`WorldQuery::IS_DENSE`]: crate::query::WorldQuery::IS_DENSE const IS_DENSE: bool = Q::IS_DENSE && F::IS_DENSE; /// Creates a new [`QuerySwitch`] of the dense variant. @@ -728,13 +732,10 @@ impl QuerySwitch { /// # Safety /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. pub unsafe fn dense(&mut self) -> &mut A { - // SAFETY: The variant of the union is checked at compile time - unsafe { - if Self::IS_DENSE { - &mut self.dense - } else { - debug_checked_unreachable() - } + if Self::IS_DENSE { + &mut self.dense + } else { + debug_checked_unreachable() } } @@ -747,13 +748,10 @@ impl QuerySwitch { /// # Safety /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. pub unsafe fn sparse(&mut self) -> &mut B { - // SAFETY: The variant of the union is checked at compile time - unsafe { - if !Self::IS_DENSE { - &mut self.sparse - } else { - debug_checked_unreachable() - } + if !Self::IS_DENSE { + &mut self.sparse + } else { + debug_checked_unreachable() } } } From 678f714d98e78011f7033ea49a35666e31f45ab9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 1 Nov 2022 00:39:33 -0700 Subject: [PATCH 5/6] Clean up docs --- crates/bevy_ecs/src/query/fetch.rs | 4 ++++ crates/bevy_ecs/src/query/iter.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index a48d870ef28d5..26c6e97806f2b 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -1549,6 +1549,7 @@ impl StorageSwitch { /// /// # Safety /// `T` must be a table component. + #[inline] pub const unsafe fn new_table(table: A) -> Self { match T::Storage::STORAGE_TYPE { StorageType::Table => Self { @@ -1565,6 +1566,7 @@ impl StorageSwitch { /// /// # Safety /// `T` must be a component. + #[inline] pub const unsafe fn new_sparse_set(sparse_set: B) -> Self { // SAFETY: The variant of the union is checked at compile time match T::Storage::STORAGE_TYPE { @@ -1584,6 +1586,7 @@ impl StorageSwitch { /// /// # Safety /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_table`]. + #[inline] pub unsafe fn table(&self) -> A { match T::Storage::STORAGE_TYPE { StorageType::Table => *self.table, @@ -1600,6 +1603,7 @@ impl StorageSwitch { /// /// # Safety /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_sparse_set`]. + #[inline] pub unsafe fn sparse_set(&self) -> B { match T::Storage::STORAGE_TYPE { StorageType::SparseSet => *self.sparse_set, diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index d96eb98574de4..d8290d322b057 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -695,6 +695,7 @@ impl QuerySwitch { /// /// # Safety /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + #[inline] pub const unsafe fn new_dense(dense: A) -> Self { if Self::IS_DENSE { Self { @@ -713,6 +714,7 @@ impl QuerySwitch { /// /// # Safety /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + #[inline] pub const unsafe fn new_sparse(sparse: B) -> Self { if !Self::IS_DENSE { Self { @@ -731,6 +733,7 @@ impl QuerySwitch { /// /// # Safety /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + #[inline] pub unsafe fn dense(&mut self) -> &mut A { if Self::IS_DENSE { &mut self.dense @@ -747,6 +750,7 @@ impl QuerySwitch { /// /// # Safety /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + #[inline] pub unsafe fn sparse(&mut self) -> &mut B { if !Self::IS_DENSE { &mut self.sparse From 43fd342a8f54cebc8fd3dcf2cfec94d14eb16827 Mon Sep 17 00:00:00 2001 From: james7132 Date: Wed, 30 Nov 2022 13:01:44 -0800 Subject: [PATCH 6/6] Fix CI --- crates/bevy_ecs/src/query/fetch.rs | 75 ++++++++++++++------------- crates/bevy_ecs/src/query/filter.rs | 10 ++-- crates/bevy_ecs/src/query/iter.rs | 78 ++++++++++++++++++++++++----- 3 files changed, 109 insertions(+), 54 deletions(-) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index e8aa32176f610..58733dad86594 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -556,7 +556,7 @@ unsafe impl WorldQuery for &T { .storages() .sparse_sets .get(component_id) - .debug_checked_unwrap() + .debug_checked_unwrap(), ), }, } @@ -739,9 +739,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { &component_id: &ComponentId, table: &'w Table, ) { - let column = table - .get_column(component_id) - .debug_checked_unwrap(); + let column = table.get_column(component_id).debug_checked_unwrap(); fetch.components = StorageSwitch::new_table(Some(( column.get_data_slice().into(), column.get_added_ticks_slice().into(), @@ -757,10 +755,8 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { ) -> Self::Item<'w> { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let (table_components, table_ticks) = fetch - .components - .table() - .debug_checked_unwrap(); + let (table_components, added_ticks, changed_ticks) = + fetch.components.table().debug_checked_unwrap(); Mut { value: table_components.get(table_row).deref_mut(), ticks: Ticks { @@ -772,7 +768,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { } } StorageType::SparseSet => { - let (component, component_ticks) = fetch + let (component, ticks) = fetch .components .sparse_set() .get_with_ticks(entity) @@ -1008,7 +1004,7 @@ pub struct ChangeTrackersFetch<'w, T: Component> { // T::Storage = TableStorage Option<( ThinSlicePtr<'w, UnsafeCell>, - ThinSlicePtr<'w, UnsafeCell> + ThinSlicePtr<'w, UnsafeCell>, )>, // T::Storage = SparseStorage &'w ComponentSparseSet, @@ -1052,7 +1048,7 @@ unsafe impl WorldQuery for ChangeTrackers { .storages() .sparse_sets .get(component_id) - .debug_checked_unwrap() + .debug_checked_unwrap(), ), }, marker: PhantomData, @@ -1089,16 +1085,10 @@ unsafe impl WorldQuery for ChangeTrackers { table: &'w Table, ) { let column = table.get_column(id).debug_checked_unwrap(); - fetch.components = StorageSwitch::new_table(Some( - column - .get_added_ticks_slice() - .get_ticks_slice() - .into(), - column - .get_changed_ticks_slice() - .get_ticks_slice() - .into(), - )); + fetch.components = StorageSwitch::new_table(Some(( + column.get_added_ticks_slice().into(), + column.get_changed_ticks_slice().into(), + ))); } #[inline(always)] @@ -1110,16 +1100,11 @@ unsafe impl WorldQuery for ChangeTrackers { match T::Storage::STORAGE_TYPE { StorageType::Table => ChangeTrackers { component_ticks: { - let (table_added, table_changed) = fetch.components.table().debug_checked_unwrap(); + let (table_added, table_changed) = + fetch.components.table().debug_checked_unwrap(); ComponentTicks { - added: - table_added - .get(table_row) - .read(), - changed: - table_changed - .get(table_row) - .read(), + added: table_added.get(table_row).read(), + changed: table_changed.get(table_row).read(), } }, marker: PhantomData, @@ -1127,7 +1112,7 @@ unsafe impl WorldQuery for ChangeTrackers { change_tick: fetch.change_tick, }, StorageType::SparseSet => ChangeTrackers { - component_ticks: *fetch + component_ticks: fetch .components .sparse_set() .get_ticks(entity) @@ -1515,7 +1500,12 @@ impl StorageSwitch { StorageType::Table => Self { table: ManuallyDrop::new(table), }, - _ => debug_checked_unreachable(), + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } } } @@ -1533,7 +1523,12 @@ impl StorageSwitch { StorageType::SparseSet => Self { sparse_set: ManuallyDrop::new(sparse_set), }, - _ => debug_checked_unreachable(), + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } } } } @@ -1552,7 +1547,12 @@ impl StorageSwitch { StorageType::Table => *self.table, // SAFETY: If T::Storage::StorageType is set to table, new_table // would have panicked instead of producing a value. - _ => debug_checked_unreachable(), + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } } } @@ -1569,7 +1569,12 @@ impl StorageSwitch { StorageType::SparseSet => *self.sparse_set, // SAFETY: If T::Storage::StorageType is set to table, new_sparse_set // would have panicked instead of producing a value. - _ => debug_checked_unreachable(), + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } } } } diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index 3f3d2ce5b0125..e6b1d19319ac4 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -2,10 +2,7 @@ use crate::{ archetype::{Archetype, ArchetypeComponentId}, component::{Component, ComponentId, ComponentStorage, StorageType, Tick}, entity::Entity, - query::{ - fetch::StorageSwitch, Access, FilteredAccess, QueryFetch, - WorldQuery, WorldQueryGats, - }, + query::{fetch::StorageSwitch, Access, DebugCheckedUnwrap, FilteredAccess, WorldQuery}, storage::{Column, ComponentSparseSet, Table}, world::World, }; @@ -506,7 +503,6 @@ macro_rules! impl_tick_filter { ) -> Self::Item<'w> { match T::Storage::STORAGE_TYPE { StorageType::Table => { -<<<<<<< HEAD fetch .components .table() @@ -517,8 +513,8 @@ macro_rules! impl_tick_filter { } StorageType::SparseSet => { let sparse_set = &fetch - .sparse_set - .debug_checked_unwrap(); + .components + .sparse_set(); $get_sparse_set(sparse_set, entity) .debug_checked_unwrap() .deref() diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index 6e650aef2bb20..5082ca1510830 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -578,10 +578,12 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, /// will be **the exact count of remaining values**. fn max_remaining(&self, tables: &'w Tables, archetypes: &'w Archetypes) -> usize { let remaining_matched: usize = if Self::IS_DENSE { - let ids = self.table_id_iter.clone(); + // SAFETY: The sparse/dense status is checked above + let ids = unsafe { self.id_iter.dense().clone() }; ids.map(|id| tables[*id].entity_count()).sum() } else { - let ids = self.archetype_id_iter.clone(); + // SAFETY: The sparse/dense status is checked above + let ids = unsafe { self.id_iter.sparse().clone() }; ids.map(|id| archetypes[*id].len()).sum() }; remaining_matched + self.current_len - self.current_index @@ -604,7 +606,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, loop { // we are on the beginning of the query, or finished processing a table, so skip to the next if self.current_index == self.current_len { - let table_id = self.id_iter.dense().next()?; + let table_id = self.id_iter.dense_mut().next()?; let table = tables.get(*table_id).debug_checked_unwrap(); // SAFETY: `table` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -634,7 +636,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, } else { loop { if self.current_index == self.current_len { - let archetype_id = self.id_iter.sparse().next()?; + let archetype_id = self.id_iter.sparse_mut().next()?; let archetype = archetypes.get(*archetype_id).debug_checked_unwrap(); // SAFETY: `archetype` and `tables` are from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -702,13 +704,16 @@ impl QuerySwitch { /// # Safety /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. #[inline] - pub const unsafe fn new_dense(dense: A) -> Self { + const unsafe fn new_dense(dense: A) -> Self { if Self::IS_DENSE { Self { dense: ManuallyDrop::new(dense), } } else { - debug_checked_unreachable() + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() } } @@ -721,13 +726,56 @@ impl QuerySwitch { /// # Safety /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. #[inline] - pub const unsafe fn new_sparse(sparse: B) -> Self { + const unsafe fn new_sparse(sparse: B) -> Self { if !Self::IS_DENSE { Self { sparse: ManuallyDrop::new(sparse), } } else { - debug_checked_unreachable() + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Fetches a reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if either `Q::IS_DENSE` and `F::IS_DENSE` + /// are not true. + /// + /// # Safety + /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + #[inline] + unsafe fn dense(&self) -> &A { + if Self::IS_DENSE { + &self.dense + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Fetches a reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if both `Q::IS_DENSE` and `F::IS_DENSE` + /// are true. + /// + /// # Safety + /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + #[inline] + unsafe fn sparse(&self) -> &B { + if !Self::IS_DENSE { + &self.sparse + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() } } @@ -740,11 +788,14 @@ impl QuerySwitch { /// # Safety /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. #[inline] - pub unsafe fn dense(&mut self) -> &mut A { + unsafe fn dense_mut(&mut self) -> &mut A { if Self::IS_DENSE { &mut self.dense } else { - debug_checked_unreachable() + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() } } @@ -757,11 +808,14 @@ impl QuerySwitch { /// # Safety /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. #[inline] - pub unsafe fn sparse(&mut self) -> &mut B { + unsafe fn sparse_mut(&mut self) -> &mut B { if !Self::IS_DENSE { &mut self.sparse } else { - debug_checked_unreachable() + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() } } }