diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index 9e62594e3c44e..58733dad86594 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -10,7 +10,7 @@ use crate::{ use bevy_ecs_macros::all_tuples; pub use bevy_ecs_macros::WorldQuery; use bevy_ptr::{ThinSlicePtr, UnsafeCellDeref}; -use std::{cell::UnsafeCell, marker::PhantomData}; +use std::{cell::UnsafeCell, marker::PhantomData, mem::ManuallyDrop}; /// Types that can be fetched from a [`World`] using a [`Query`]. /// @@ -512,11 +512,14 @@ unsafe impl WorldQuery for Entity { unsafe impl ReadOnlyWorldQuery for Entity {} #[doc(hidden)] -pub struct ReadFetch<'w, T> { - // T::Storage = TableStorage - table_components: Option>>, - // T::Storage = SparseStorage - sparse_set: Option<&'w ComponentSparseSet>, +pub struct ReadFetch<'w, T: Component> { + components: StorageSwitch< + T, + // T::Storage = TableStorage + Option>>, + // T::Storage = SparseStorage + &'w ComponentSparseSet, + >, } /// SAFETY: `Self` is the same as `Self::ReadOnly` @@ -546,21 +549,22 @@ unsafe impl WorldQuery for &T { _change_tick: u32, ) -> ReadFetch<'w, T> { ReadFetch { - table_components: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { - world - .storages() - .sparse_sets - .get(component_id) - .debug_checked_unwrap() - }), + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( + world + .storages() + .sparse_sets + .get(component_id) + .debug_checked_unwrap(), + ), + }, } } unsafe fn clone_fetch<'w>(fetch: &Self::Fetch<'w>) -> Self::Fetch<'w> { ReadFetch { - table_components: fetch.table_components, - sparse_set: fetch.sparse_set, + components: fetch.components, } } @@ -582,13 +586,13 @@ unsafe impl WorldQuery for &T { &component_id: &ComponentId, table: &'w Table, ) { - fetch.table_components = Some( + fetch.components = StorageSwitch::new_table(Some( table .get_column(component_id) .debug_checked_unwrap() .get_data_slice() .into(), - ); + )); } #[inline(always)] @@ -599,13 +603,14 @@ unsafe impl WorldQuery for &T { ) -> Self::Item<'w> { match T::Storage::STORAGE_TYPE { StorageType::Table => fetch - .table_components + .components + .table() .debug_checked_unwrap() .get(table_row) .deref(), StorageType::SparseSet => fetch - .sparse_set - .debug_checked_unwrap() + .components + .sparse_set() .get(entity) .debug_checked_unwrap() .deref(), @@ -650,16 +655,18 @@ unsafe impl WorldQuery for &T { unsafe impl ReadOnlyWorldQuery for &T {} #[doc(hidden)] -pub struct WriteFetch<'w, T> { - // T::Storage = TableStorage - table_data: Option<( - ThinSlicePtr<'w, UnsafeCell>, - ThinSlicePtr<'w, UnsafeCell>, - ThinSlicePtr<'w, UnsafeCell>, - )>, - // T::Storage = SparseStorage - sparse_set: Option<&'w ComponentSparseSet>, - +pub struct WriteFetch<'w, T: Component> { + components: StorageSwitch< + T, + // T::Storage = TableStorage + Option<( + ThinSlicePtr<'w, UnsafeCell>, + ThinSlicePtr<'w, UnsafeCell>, + ThinSlicePtr<'w, UnsafeCell>, + )>, + // T::Storage = SparseStorage + &'w ComponentSparseSet, + >, last_change_tick: u32, change_tick: u32, } @@ -691,14 +698,16 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { change_tick: u32, ) -> WriteFetch<'w, T> { WriteFetch { - table_data: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { - world - .storages() - .sparse_sets - .get(component_id) - .debug_checked_unwrap() - }), + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( + world + .storages() + .sparse_sets + .get(component_id) + .debug_checked_unwrap(), + ), + }, last_change_tick, change_tick, } @@ -706,8 +715,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { unsafe fn clone_fetch<'w>(fetch: &Self::Fetch<'w>) -> Self::Fetch<'w> { WriteFetch { - table_data: fetch.table_data, - sparse_set: fetch.sparse_set, + components: fetch.components, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, } @@ -732,11 +740,11 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { table: &'w Table, ) { let column = table.get_column(component_id).debug_checked_unwrap(); - fetch.table_data = Some(( + fetch.components = StorageSwitch::new_table(Some(( column.get_data_slice().into(), column.get_added_ticks_slice().into(), column.get_changed_ticks_slice().into(), - )); + ))); } #[inline(always)] @@ -748,7 +756,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { match T::Storage::STORAGE_TYPE { StorageType::Table => { let (table_components, added_ticks, changed_ticks) = - fetch.table_data.debug_checked_unwrap(); + fetch.components.table().debug_checked_unwrap(); Mut { value: table_components.get(table_row).deref_mut(), ticks: Ticks { @@ -761,8 +769,8 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { } StorageType::SparseSet => { let (component, ticks) = fetch - .sparse_set - .debug_checked_unwrap() + .components + .sparse_set() .get_with_ticks(entity) .debug_checked_unwrap(); Mut { @@ -990,13 +998,17 @@ impl ChangeTrackers { } #[doc(hidden)] -pub struct ChangeTrackersFetch<'w, T> { - // T::Storage = TableStorage - table_added: Option>>, - table_changed: Option>>, - // T::Storage = SparseStorage - sparse_set: Option<&'w ComponentSparseSet>, - +pub struct ChangeTrackersFetch<'w, T: Component> { + components: StorageSwitch< + T, + // T::Storage = TableStorage + Option<( + ThinSlicePtr<'w, UnsafeCell>, + ThinSlicePtr<'w, UnsafeCell>, + )>, + // T::Storage = SparseStorage + &'w ComponentSparseSet, + >, marker: PhantomData, last_change_tick: u32, change_tick: u32, @@ -1029,15 +1041,16 @@ unsafe impl WorldQuery for ChangeTrackers { change_tick: u32, ) -> ChangeTrackersFetch<'w, T> { ChangeTrackersFetch { - table_added: None, - table_changed: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { - world - .storages() - .sparse_sets - .get(component_id) - .debug_checked_unwrap() - }), + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( + world + .storages() + .sparse_sets + .get(component_id) + .debug_checked_unwrap(), + ), + }, marker: PhantomData, last_change_tick, change_tick, @@ -1046,9 +1059,7 @@ unsafe impl WorldQuery for ChangeTrackers { unsafe fn clone_fetch<'w>(fetch: &Self::Fetch<'w>) -> Self::Fetch<'w> { ChangeTrackersFetch { - table_added: fetch.table_added, - table_changed: fetch.table_changed, - sparse_set: fetch.sparse_set, + components: fetch.components, marker: fetch.marker, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, @@ -1074,8 +1085,10 @@ unsafe impl WorldQuery for ChangeTrackers { table: &'w Table, ) { let column = table.get_column(id).debug_checked_unwrap(); - fetch.table_added = Some(column.get_added_ticks_slice().into()); - fetch.table_changed = Some(column.get_changed_ticks_slice().into()); + fetch.components = StorageSwitch::new_table(Some(( + column.get_added_ticks_slice().into(), + column.get_changed_ticks_slice().into(), + ))); } #[inline(always)] @@ -1087,17 +1100,11 @@ unsafe impl WorldQuery for ChangeTrackers { match T::Storage::STORAGE_TYPE { StorageType::Table => ChangeTrackers { component_ticks: { + let (table_added, table_changed) = + fetch.components.table().debug_checked_unwrap(); ComponentTicks { - added: fetch - .table_added - .debug_checked_unwrap() - .get(table_row) - .read(), - changed: fetch - .table_changed - .debug_checked_unwrap() - .get(table_row) - .read(), + added: table_added.get(table_row).read(), + changed: table_changed.get(table_row).read(), } }, marker: PhantomData, @@ -1106,8 +1113,8 @@ unsafe impl WorldQuery for ChangeTrackers { }, StorageType::SparseSet => ChangeTrackers { component_ticks: fetch - .sparse_set - .debug_checked_unwrap() + .components + .sparse_set() .get_ticks(entity) .debug_checked_unwrap(), marker: PhantomData, @@ -1470,3 +1477,122 @@ unsafe impl WorldQuery for NopWorldQuery { /// SAFETY: `NopFetch` never accesses any data unsafe impl ReadOnlyWorldQuery for NopWorldQuery {} + +/// A compile-time checked union of two different types that differs based +/// on the storage type of a given Component. +pub(super) union StorageSwitch { + table: ManuallyDrop, + sparse_set: ManuallyDrop, + marker: PhantomData, +} + +impl StorageSwitch { + /// Creates a new [`StorageSwitch`] using a table variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a table component. + /// + /// # Safety + /// `T` must be a table component. + #[inline] + pub const unsafe fn new_table(table: A) -> Self { + match T::Storage::STORAGE_TYPE { + StorageType::Table => Self { + table: ManuallyDrop::new(table), + }, + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + } + + /// Creates a new [`StorageSwitch`] using a sparse set variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a sparse set component. + /// + /// # Safety + /// `T` must be a component. + #[inline] + pub const unsafe fn new_sparse_set(sparse_set: B) -> Self { + // SAFETY: The variant of the union is checked at compile time + match T::Storage::STORAGE_TYPE { + StorageType::SparseSet => Self { + sparse_set: ManuallyDrop::new(sparse_set), + }, + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + } +} + +impl StorageSwitch { + /// Fetches the internal value as a table variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a table component. + /// + /// # Safety + /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_table`]. + #[inline] + pub unsafe fn table(&self) -> A { + match T::Storage::STORAGE_TYPE { + StorageType::Table => *self.table, + // SAFETY: If T::Storage::StorageType is set to table, new_table + // would have panicked instead of producing a value. + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + } + + /// Fetches the internal value as a sparse set variant. + /// + /// # Panics + /// This will panic on debug builds if `T` is not a sparse set component. + /// + /// # Safety + /// The [`StorageSwitch`] must be made via [`StorageSwitch::new_sparse_set`]. + #[inline] + pub unsafe fn sparse_set(&self) -> B { + match T::Storage::STORAGE_TYPE { + StorageType::SparseSet => *self.sparse_set, + // SAFETY: If T::Storage::StorageType is set to table, new_sparse_set + // would have panicked instead of producing a value. + _ => { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + } +} + +impl Clone for StorageSwitch { + fn clone(&self) -> Self { + // SAFETY: The variant of the union is checked at compile time + unsafe { + match T::Storage::STORAGE_TYPE { + StorageType::Table => Self { + table: self.table.clone(), + }, + StorageType::SparseSet => Self { + sparse_set: self.sparse_set.clone(), + }, + } + } + } +} + +impl Copy for StorageSwitch {} diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index a84d74f8075e3..e6b1d19319ac4 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -2,7 +2,7 @@ use crate::{ archetype::{Archetype, ArchetypeComponentId}, component::{Component, ComponentId, ComponentStorage, StorageType, Tick}, entity::Entity, - query::{Access, DebugCheckedUnwrap, FilteredAccess, WorldQuery}, + query::{fetch::StorageSwitch, Access, DebugCheckedUnwrap, FilteredAccess, WorldQuery}, storage::{Column, ComponentSparseSet, Table}, world::World, }; @@ -413,10 +413,13 @@ macro_rules! impl_tick_filter { #[doc(hidden)] $(#[$fetch_meta])* - pub struct $fetch_name<'w, T> { - table_ticks: Option< ThinSlicePtr<'w, UnsafeCell>>, + pub struct $fetch_name<'w, T: Component> { + components: StorageSwitch< + T, + Option>>, + &'w ComponentSparseSet, + >, marker: PhantomData, - sparse_set: Option<&'w ComponentSparseSet>, last_change_tick: u32, change_tick: u32, } @@ -434,14 +437,15 @@ macro_rules! impl_tick_filter { unsafe fn init_fetch<'w>(world: &'w World, &id: &ComponentId, last_change_tick: u32, change_tick: u32) -> Self::Fetch<'w> { Self::Fetch::<'w> { - table_ticks: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet) - .then(|| { + components: match T::Storage::STORAGE_TYPE { + StorageType::Table => StorageSwitch::new_table(None), + StorageType::SparseSet => StorageSwitch::new_sparse_set( world.storages() .sparse_sets .get(id) .debug_checked_unwrap() - }), + ), + }, marker: PhantomData, last_change_tick, change_tick, @@ -452,8 +456,7 @@ macro_rules! impl_tick_filter { fetch: &Self::Fetch<'w>, ) -> Self::Fetch<'w> { $fetch_name { - table_ticks: fetch.table_ticks, - sparse_set: fetch.sparse_set, + components: fetch.components, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, marker: PhantomData, @@ -475,13 +478,9 @@ macro_rules! impl_tick_filter { &component_id: &ComponentId, table: &'w Table ) { - fetch.table_ticks = Some( - $get_slice( - &table - .get_column(component_id) - .debug_checked_unwrap() - ).into(), - ); + fetch.components = StorageSwitch::new_table(Some( + $get_slice(&table.get_column(component_id).debug_checked_unwrap()).into() + )); } #[inline] @@ -505,7 +504,8 @@ macro_rules! impl_tick_filter { match T::Storage::STORAGE_TYPE { StorageType::Table => { fetch - .table_ticks + .components + .table() .debug_checked_unwrap() .get(table_row) .deref() @@ -513,8 +513,8 @@ macro_rules! impl_tick_filter { } StorageType::SparseSet => { let sparse_set = &fetch - .sparse_set - .debug_checked_unwrap(); + .components + .sparse_set(); $get_sparse_set(sparse_set, entity) .debug_checked_unwrap() .deref() diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index c5f1bab22095d..5082ca1510830 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -2,10 +2,16 @@ use crate::{ archetype::{ArchetypeEntity, ArchetypeId, Archetypes}, entity::{Entities, Entity}, prelude::World, + ptr::ThinSlicePtr, query::{ArchetypeFilter, DebugCheckedUnwrap, QueryState, WorldQuery}, storage::{TableId, Tables}, }; -use std::{borrow::Borrow, iter::FusedIterator, marker::PhantomData, mem::MaybeUninit}; +use std::{ + borrow::Borrow, + iter::FusedIterator, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, +}; use super::ReadOnlyWorldQuery; @@ -455,10 +461,8 @@ impl<'w, 's, Q: ReadOnlyWorldQuery, F: ReadOnlyWorldQuery, const K: usize> Fused } struct QueryIterationCursor<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> { - table_id_iter: std::slice::Iter<'s, TableId>, - archetype_id_iter: std::slice::Iter<'s, ArchetypeId>, - table_entities: &'w [Entity], - archetype_entities: &'w [ArchetypeEntity], + id_iter: QuerySwitch, std::slice::Iter<'s, ArchetypeId>>, + entities: QuerySwitch, ThinSlicePtr<'w, ArchetypeEntity>>, fetch: Q::Fetch<'w>, filter: F::Fetch<'w>, // length of the table table or length of the archetype, depending on whether both `Q`'s and `F`'s fetches are dense @@ -477,10 +481,8 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, /// `archetype_index` or `table_row` to be alive at the same time. unsafe fn clone_cursor(&self) -> Self { Self { - table_id_iter: self.table_id_iter.clone(), - archetype_id_iter: self.archetype_id_iter.clone(), - table_entities: self.table_entities, - archetype_entities: self.archetype_entities, + id_iter: self.id_iter.clone(), + entities: self.entities.clone(), // SAFETY: upheld by caller invariants fetch: Q::clone_fetch(&self.fetch), filter: F::clone_fetch(&self.filter), @@ -501,8 +503,11 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, change_tick: u32, ) -> Self { QueryIterationCursor { - table_id_iter: [].iter(), - archetype_id_iter: [].iter(), + id_iter: if Self::IS_DENSE { + QuerySwitch::new_dense([].iter()) + } else { + QuerySwitch::new_sparse([].iter()) + }, ..Self::init(world, query_state, last_change_tick, change_tick) } } @@ -525,13 +530,21 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, last_change_tick, change_tick, ); + let table_entities: &[Entity] = &[]; + let archetype_entities: &[ArchetypeEntity] = &[]; QueryIterationCursor { fetch, filter, - table_entities: &[], - archetype_entities: &[], - table_id_iter: query_state.matched_table_ids.iter(), - archetype_id_iter: query_state.matched_archetype_ids.iter(), + id_iter: if Self::IS_DENSE { + QuerySwitch::new_dense(query_state.matched_table_ids.iter()) + } else { + QuerySwitch::new_sparse(query_state.matched_archetype_ids.iter()) + }, + entities: if Self::IS_DENSE { + QuerySwitch::new_dense(table_entities.into()) + } else { + QuerySwitch::new_sparse(archetype_entities.into()) + }, current_len: 0, current_index: 0, phantom: PhantomData, @@ -544,10 +557,10 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, if self.current_index > 0 { let index = self.current_index - 1; if Self::IS_DENSE { - let entity = self.table_entities.get_unchecked(index); + let entity = self.entities.dense().get(index); Some(Q::fetch(&mut self.fetch, *entity, index)) } else { - let archetype_entity = self.archetype_entities.get_unchecked(index); + let archetype_entity = self.entities.sparse().get(index); Some(Q::fetch( &mut self.fetch, archetype_entity.entity(), @@ -565,10 +578,12 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, /// will be **the exact count of remaining values**. fn max_remaining(&self, tables: &'w Tables, archetypes: &'w Archetypes) -> usize { let remaining_matched: usize = if Self::IS_DENSE { - let ids = self.table_id_iter.clone(); + // SAFETY: The sparse/dense status is checked above + let ids = unsafe { self.id_iter.dense().clone() }; ids.map(|id| tables[*id].entity_count()).sum() } else { - let ids = self.archetype_id_iter.clone(); + // SAFETY: The sparse/dense status is checked above + let ids = unsafe { self.id_iter.sparse().clone() }; ids.map(|id| archetypes[*id].len()).sum() }; remaining_matched + self.current_len - self.current_index @@ -591,13 +606,13 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, loop { // we are on the beginning of the query, or finished processing a table, so skip to the next if self.current_index == self.current_len { - let table_id = self.table_id_iter.next()?; + let table_id = self.id_iter.dense_mut().next()?; let table = tables.get(*table_id).debug_checked_unwrap(); // SAFETY: `table` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with Q::set_table(&mut self.fetch, &query_state.fetch_state, table); F::set_table(&mut self.filter, &query_state.filter_state, table); - self.table_entities = table.entities(); + self.entities = QuerySwitch::new_dense(table.entities().into()); self.current_len = table.entity_count(); self.current_index = 0; continue; @@ -605,7 +620,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // SAFETY: set_table was called prior. // `current_index` is a table row in range of the current table, because if it was not, then the if above would have been executed. - let entity = self.table_entities.get_unchecked(self.current_index); + let entity = self.entities.dense().get(self.current_index); if !F::filter_fetch(&mut self.filter, *entity, self.current_index) { self.current_index += 1; continue; @@ -621,7 +636,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, } else { loop { if self.current_index == self.current_len { - let archetype_id = self.archetype_id_iter.next()?; + let archetype_id = self.id_iter.sparse_mut().next()?; let archetype = archetypes.get(*archetype_id).debug_checked_unwrap(); // SAFETY: `archetype` and `tables` are from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -633,7 +648,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, archetype, table, ); - self.archetype_entities = archetype.entities(); + self.entities = QuerySwitch::new_sparse(archetype.entities().into()); self.current_len = archetype.len(); self.current_index = 0; continue; @@ -641,7 +656,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // SAFETY: set_archetype was called prior. // `current_index` is an archetype index row in range of the current archetype, because if it was not, then the if above would have been executed. - let archetype_entity = self.archetype_entities.get_unchecked(self.current_index); + let archetype_entity = self.entities.sparse().get(self.current_index); if !F::filter_fetch( &mut self.filter, archetype_entity.entity(), @@ -664,3 +679,160 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, } } } + +/// A compile-time checked union of two different types that differs based +/// whether a fetch is dense or not. +union QuerySwitch { + dense: ManuallyDrop, + sparse: ManuallyDrop, + marker: PhantomData<(Q, F)>, +} + +impl QuerySwitch { + /// Whether the corresponding query is using dense iteration or not. + /// For more information, see [`WorldQuery::IS_DENSE`] + /// + /// [`WorldQuery::IS_DENSE`]: crate::query::WorldQuery::IS_DENSE + const IS_DENSE: bool = Q::IS_DENSE && F::IS_DENSE; + + /// Creates a new [`QuerySwitch`] of the dense variant. + /// + /// # Panics + /// Will panic in debug mode if either `Q::IS_DENSE` and `F::IS_DENSE` + /// are not true. + /// + /// # Safety + /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + #[inline] + const unsafe fn new_dense(dense: A) -> Self { + if Self::IS_DENSE { + Self { + dense: ManuallyDrop::new(dense), + } + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Creates a new [`QuerySwitch`] of the sparse variant. + /// + /// # Panics + /// Will panic in debug mode if both `Q::IS_DENSE` and `F::IS_DENSE` + /// are true. + /// + /// # Safety + /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + #[inline] + const unsafe fn new_sparse(sparse: B) -> Self { + if !Self::IS_DENSE { + Self { + sparse: ManuallyDrop::new(sparse), + } + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Fetches a reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if either `Q::IS_DENSE` and `F::IS_DENSE` + /// are not true. + /// + /// # Safety + /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + #[inline] + unsafe fn dense(&self) -> &A { + if Self::IS_DENSE { + &self.dense + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Fetches a reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if both `Q::IS_DENSE` and `F::IS_DENSE` + /// are true. + /// + /// # Safety + /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + #[inline] + unsafe fn sparse(&self) -> &B { + if !Self::IS_DENSE { + &self.sparse + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Fetches a mutable reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if either `Q::IS_DENSE` and `F::IS_DENSE` + /// are not true. + /// + /// # Safety + /// Both `Q::IS_DENSE` and `F::IS_DENSE` must be true. + #[inline] + unsafe fn dense_mut(&mut self) -> &mut A { + if Self::IS_DENSE { + &mut self.dense + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } + + /// Fetches a mutable reference to the dense variant. + /// + /// # Panics + /// Will panic in debug mode if both `Q::IS_DENSE` and `F::IS_DENSE` + /// are true. + /// + /// # Safety + /// Either `Q::IS_DENSE` or `F::IS_DENSE` must be false. + #[inline] + unsafe fn sparse_mut(&mut self) -> &mut B { + if !Self::IS_DENSE { + &mut self.sparse + } else { + #[cfg(debug_assertions)] + unreachable!(); + #[cfg(not(debug_assertions))] + std::hint::unreachable_unchecked() + } + } +} + +impl Clone for QuerySwitch { + fn clone(&self) -> Self { + // SAFETY: The variant of the union is checked at compile time + unsafe { + if Self::IS_DENSE { + Self { + dense: self.dense.clone(), + } + } else { + Self { + sparse: self.sparse.clone(), + } + } + } + } +}