Make mocking picking events for bevy_ui easier

crates/bevy_picking/src/input.rs

@@ -79,7 +79,7 @@ impl Default for PointerInputPlugin {
 impl Plugin for PointerInputPlugin {
     fn build(&self, app: &mut App) {
         app.insert_resource(*self)
-            .add_systems(Startup, spawn_mouse_pointer)
+            .add_systems(Startup, (spawn_mouse_pointer, spawn_focus_pointer))
             .add_systems(
                 First,
                 (
@@ -103,6 +103,11 @@ pub fn spawn_mouse_pointer(mut commands: Commands) {
     commands.spawn(PointerId::Mouse);
 }
 
+/// Spawns the default focus pointer.
+pub fn spawn_focus_pointer(mut commands: Commands) {
+    commands.spawn(PointerId::Focus);
+}
+
 /// Sends mouse pointer events to be processed by the core plugin
 pub fn mouse_pick_events(
     // Input

crates/bevy_picking/src/pointer.rs

@@ -34,6 +34,10 @@ pub enum PointerId {
     Mouse,
     /// A touch input, usually numbered by window touch events from `winit`.
     Touch(u64),
+    /// An emulated pointer linked to the focused entity.
+    ///
+    /// Generally triggered by the `Enter` key or an `A` input on a gamepad.
+    Focus,
     /// A custom, uniquely identified pointer. Useful for mocking inputs or implementing a software
     /// controlled cursor.
     #[reflect(ignore)]
@@ -197,13 +201,15 @@ impl PointerLocation {
 /// The location of a pointer, including the current [`NormalizedRenderTarget`], and the x/y
 /// position of the pointer on this render target.
 ///
+/// This is stored as a [`PointerLocation`] component on the pointer entity.
+///
 /// Note that:
 /// - a pointer can move freely between render targets
 /// - a pointer is not associated with a [`Camera`] because multiple cameras can target the same
 ///   render target. It is up to picking backends to associate a Pointer's `Location` with a
 ///   specific `Camera`, if any.
-#[derive(Debug, Clone, Component, Reflect, PartialEq)]
-#[reflect(Component, Debug, PartialEq)]
+#[derive(Debug, Clone, Reflect, PartialEq)]
+#[reflect(Debug, PartialEq)]
 pub struct Location {
     /// The [`NormalizedRenderTarget`] associated with the pointer, usually a window.
     pub target: NormalizedRenderTarget,
@@ -240,6 +246,8 @@ impl Location {
 }
 
 /// Types of actions that can be taken by pointers.
+///
+/// These are sent as the payload of [`PointerInput`] events.
 #[derive(Debug, Clone, Copy, Reflect)]
 pub enum PointerAction {
     /// A button has been pressed on the pointer.
@@ -259,13 +267,25 @@ pub enum PointerAction {
 }
 
 /// An input event effecting a pointer.
+///
+/// These events are generated from user input in the [`PointerInputPlugin`](crate::input::PointerInputPlugin)
+/// and are read by the [`PointerInput::receive`] system
+/// to modify the state of existing pointer entities.
 #[derive(Event, Debug, Clone, Reflect)]
 pub struct PointerInput {
     /// The id of the pointer.
+    ///
+    /// Used to identify which pointer entity to update.
+    /// If no match is found, the event is ignored: no new pointer entity is created.
     pub pointer_id: PointerId,
-    /// The location of the pointer. For [[`PointerAction::Moved`]], this is the location after the movement.
+    /// The location of the pointer. For [`PointerAction::Moved`], this is the location after the movement.
+    ///
+    /// Defines exactly where, on the [`NormalizedRenderTarget`] that the pointer event came from,
+    /// the event occurred.
     pub location: Location,
     /// The action that the event describes.
+    ///
+    /// This is the action that the pointer took, such as pressing a button or moving.
     pub action: PointerAction,
 }
 
@@ -302,6 +322,8 @@ impl PointerInput {
     }
 
     /// Updates pointer entities according to the input events.
+    ///
+    /// Entities are matched by their [`PointerId`]. If no match is found, the event is ignored.
     pub fn receive(
         mut events: EventReader<PointerInput>,
         mut pointers: Query<(&PointerId, &mut PointerLocation, &mut PointerPress)>,

crates/bevy_ui/src/picking_backend.rs

@@ -23,14 +23,18 @@
 
 use crate::{focus::pick_rounded_rect, prelude::*, UiStack};
 use bevy_app::prelude::*;
-use bevy_ecs::{prelude::*, query::QueryData};
-use bevy_math::{Rect, Vec2};
+use bevy_ecs::{prelude::*, query::QueryData, system::SystemParam};
+use bevy_math::{Rect, Vec2, Vec3Swizzles};
 use bevy_render::prelude::*;
 use bevy_transform::prelude::*;
 use bevy_utils::HashMap;
 use bevy_window::PrimaryWindow;
 
-use bevy_picking::backend::prelude::*;
+use bevy_picking::{
+    backend::prelude::*,
+    pointer::{Location, PointerAction, PointerInput},
+};
+use thiserror::Error;
 
 /// A plugin that adds picking support for UI nodes.
 #[derive(Clone)]
@@ -58,6 +62,9 @@ pub struct NodeQuery {
 ///
 /// Bevy's [`UiStack`] orders all nodes in the order they will be rendered, which is the same order
 /// we need for determining picking.
+///
+/// Like all picking backends, this system reads the [`PointerId`] and [`PointerLocation`] components,
+/// and produces [`PointerHits`] events.
 pub fn ui_picking(
     pointers: Query<(&PointerId, &PointerLocation)>,
     camera_query: Query<(Entity, &Camera, Has<IsDefaultUiCamera>)>,
@@ -218,3 +225,112 @@ pub fn ui_picking(
         output.send(PointerHits::new(*pointer, picks, order));
     }
 }
+
+/// A [`SystemParam`] for realistically simulating/mocking pointer events on UI nodes.
+#[derive(SystemParam)]
+pub struct EmulateNodePointerEvents<'w, 's> {
+    /// Looks up information about the node that the pointer event should be simulated on.
+    pub node_query: Query<'w, 's, (&'static GlobalTransform, Option<&'static TargetCamera>)>,
+    /// Tries to find the default UI camera
+    pub default_ui_camera: DefaultUiCamera<'w, 's>,
+    /// Tries to find a primary window entity.
+    pub primary_window_query: Query<'w, 's, Entity, With<PrimaryWindow>>,
+    /// Looks up the required camera information.
+    pub camera_query: Query<'w, 's, &'static Camera>,
+    /// Writes the pointer events to the world.
+    pub pointer_input_events: EventWriter<'w, PointerInput>,
+}
+
+impl<'w, 's> EmulateNodePointerEvents<'w, 's> {
+    /// Simulate a [`Pointer`](bevy_picking::events::Pointer) event,
+    /// at the origin of the provided UI node entity.
+    ///
+    /// The entity that represents the [`PointerId`] provided should already exist,
+    /// as this method does not create it.
+    ///
+    /// Under the hood, this generates [`PointerInput`] events,
+    /// which is read by the [`PointerInput::receive`] system to modify existing pointer entities,
+    /// and ultimately then processed into UI events by the [`ui_picking`] system.
+    ///
+    /// When using [`UiPlugin`](crate::UiPlugin), that system runs in the [`PreUpdate`] schedule,
+    /// under the [`PickSet::Backend`] set.
+    /// To ensure that these events are seen at the right time,
+    /// you should generally call this method in systems scheduled during [`First`],
+    /// as part of the [`PickSet::Input`] system set.
+    ///
+    /// # Warning
+    ///
+    /// If the node is not pickable, or is blocked by a higher node,
+    /// these events may not have any effect, even if sent correctly!
+    pub fn emulate_pointer(
+        &mut self,
+        pointer_id: PointerId,
+        pointer_action: PointerAction,
+        entity: Entity,
+    ) -> Result<(), SimulatedNodePointerError> {
+        // Look up the node we're trying to send a pointer event to
+        let Ok((global_transform, maybe_target_camera)) = self.node_query.get(entity) else {
+            return Err(SimulatedNodePointerError::NodeNotFound(entity));
+        };
+
+        // Figure out which camera this node is associated with
+        let camera_entity = match maybe_target_camera {
+            Some(explicit_target_camera) => explicit_target_camera.entity(),
+            // Fall back to the default UI camera
+            None => match self.default_ui_camera.get() {
+                Some(default_camera_entity) => default_camera_entity,
+                None => return Err(SimulatedNodePointerError::NoCameraFound),
+            },
+        };
+
+        // Find the primary window, needed to normalize the render target
+        // If we find 0 or 2+ primary windows, treat it as if none were found
+        let maybe_primary_window_entity = self.primary_window_query.get_single().ok();
+
+        // Generate the correct render target for the pointer
+        let Ok(camera) = self.camera_query.get(camera_entity) else {
+            return Err(SimulatedNodePointerError::NoCameraFound);
+        };
+
+        let Some(target) = camera.target.normalize(maybe_primary_window_entity) else {
+            return Err(SimulatedNodePointerError::CouldNotComputeRenderTarget);
+        };
+
+        // Calculate the pointer position in the render target
+        // For UI nodes, their final position is stored on their global transform,
+        // in pixels, with the origin at the top-left corner of the camera's viewport.
+        let position = global_transform.translation().xy();
+
+        let pointer_location = Location { target, position };
+
+        self.pointer_input_events.send(PointerInput {
+            pointer_id,
+            location: pointer_location,
+            action: pointer_action,
+        });
+
+        Ok(())
+    }
+}
+
+/// An error returned by [`EmulateNodePointerEvents`].
+#[derive(Debug, PartialEq, Clone, Error)]
+pub enum SimulatedNodePointerError {
+    /// The entity provided could not be found.
+    ///
+    /// It must have a [`GlobalTransform`] component,
+    /// and should have a [`Node`] component.
+    #[error("The entity {0:?} could not be found.")]
+    NodeNotFound(Entity),
+    /// The camera associated with the node could not be found.
+    ///
+    /// Did you forget to spawn a camera entity with the [`Camera`] component?
+    ///
+    /// The [`TargetCamera`] component can be used to associate a camera with a node,
+    /// but if it is not present, the [`DefaultUiCamera`] will be used.
+    #[error("No camera could be found for the node.")]
+    NoCameraFound,
+    /// The [`NormalizedRenderTarget`](bevy_render::camera::NormalizedRenderTarget) could not be computed.
+    #[error("Could not compute the normalized render target for the camera.")]
+    CouldNotComputeRenderTarget,
+}

examples/ui/directional_navigation.rs

@@ -12,13 +12,13 @@ use bevy::{
         },
         InputDispatchPlugin, InputFocus, InputFocusVisible,
     },
-    math::{CompassOctant, FloatOrd},
+    math::CompassOctant,
     picking::{
-        backend::HitData,
-        pointer::{Location, PointerId},
+        pointer::{PointerAction, PointerId, PressDirection},
+        PickSet,
     },
     prelude::*,
-    render::camera::NormalizedRenderTarget,
+    ui::picking_backend::EmulateNodePointerEvents,
     utils::{HashMap, HashSet},
 };
 
@@ -39,13 +39,21 @@ fn main() {
         // Input is generally handled during PreUpdate
         // We're turning inputs into actions first, then using those actions to determine navigation
         .add_systems(PreUpdate, (process_inputs, navigate).chain())
+        // We're simulating pointer clicks on the focused button
+        // and are running this system at the same time as the other pointer input systems
+        .add_systems(
+            First,
+            (
+                move_pointer_to_focused_element,
+                interact_with_focused_button,
+            )
+                .in_set(PickSet::Input),
+        )
         .add_systems(
             Update,
             (
                 // We need to show which button is currently focused
                 highlight_focused_element,
-                // Pressing the "Interact" button while we have a focused element should simulate a click
-                interact_with_focused_button,
                 // We're doing a tiny animation when the button is interacted with,
                 // so we need a timer and a polling mechanism to reset it
                 reset_button_after_interaction,
@@ -354,6 +362,26 @@ fn navigate(action_state: Res<ActionState>, mut directional_navigation: Directio
     }
 }
 
+// We need to update the location of the focused element
+// so that our virtual pointer presses are sent to the correct location.
+//
+// Doing this in an ordinary system (rather than just on navigation change) ensures that it is
+// initialized correctly, and is updated if the focused element moves or is scaled for any reason.
+fn move_pointer_to_focused_element(
+    input_focus: Res<InputFocus>,
+    mut emulate_pointer_events: EmulateNodePointerEvents,
+) {
+    if let Some(focused_entity) = input_focus.0 {
+        emulate_pointer_events
+            .emulate_pointer(
+                PointerId::Focus,
+                PointerAction::Moved { delta: Vec2::ZERO },
+                focused_entity,
+            )
+            .unwrap_or_else(|e| warn!("Failed to move pointer: {e}"));
+    }
+}
+
 fn highlight_focused_element(
     input_focus: Res<InputFocus>,
     // While this isn't strictly needed for the example,
@@ -372,46 +400,42 @@ fn highlight_focused_element(
     }
 }
 
-// By sending a Pointer<Pressed> trigger rather than directly handling button-like interactions,
-// we can unify our handling of pointer and keyboard/gamepad interactions
+// We're emulating a pointer event sent to the focused button,
+// which will be picked up just like any mouse or touch input!
 fn interact_with_focused_button(
     action_state: Res<ActionState>,
     input_focus: Res<InputFocus>,
-    mut commands: Commands,
+    mut emulate_pointer_events: EmulateNodePointerEvents,
 ) {
     if action_state
         .pressed_actions
         .contains(&DirectionalNavigationAction::Select)
     {
         if let Some(focused_entity) = input_focus.0 {
-            commands.trigger_targets(
-                Pointer::<Pressed> {
-                    target: focused_entity,
-                    // We're pretending that we're a mouse
-                    pointer_id: PointerId::Mouse,
-                    // This field isn't used, so we're just setting it to a placeholder value
-                    pointer_location: Location {
-                        target: NormalizedRenderTarget::Image(
-                            bevy_render::camera::ImageRenderTarget {
-                                handle: Handle::default(),
-                                scale_factor: FloatOrd(1.0),
-                            },
-                        ),
-                        position: Vec2::ZERO,
+            emulate_pointer_events
+                .emulate_pointer(
+                    PointerId::Focus,
+                    PointerAction::Pressed {
+                        direction: PressDirection::Pressed,
+                        button: PointerButton::Primary,
                     },
-                    event: Pressed {
+                    focused_entity,
+                )
+                .unwrap_or_else(|e| warn!("Failed to press pointer: {e}"));
+        }
+    } else {
+        // If the button was not pressed, we simulate a release event
+        if let Some(focused_entity) = input_focus.0 {
+            emulate_pointer_events
+                .emulate_pointer(
+                    PointerId::Focus,
+                    PointerAction::Pressed {
+                        direction: PressDirection::Released,
                         button: PointerButton::Primary,
-                        // This field isn't used, so we're just setting it to a placeholder value
-                        hit: HitData {
-                            camera: Entity::PLACEHOLDER,
-                            depth: 0.0,
-                            position: None,
-                            normal: None,
-                        },
                     },
-                },
-                focused_entity,
-            );
+                    focused_entity,
+                )
+                .unwrap_or_else(|e| warn!("Failed to release pointer: {e}"));
         }
     }
 }