switch to sending render world back and forth with channels

Author: hymm

crates/bevy_app/src/app.rs

@@ -10,7 +10,6 @@ use bevy_ecs::{
     system::Resource,
     world::World,
 };
-use bevy_tasks::{ComputeTaskPool, TaskPool};
 use bevy_utils::{tracing::debug, HashMap, HashSet};
 use std::fmt::Debug;
 
@@ -74,7 +73,6 @@ pub struct App {
     sub_apps: HashMap<AppLabelId, SubApp>,
     plugin_registry: Vec<Box<dyn Plugin>>,
     plugin_name_added: HashSet<String>,
-    run_once: bool,
 }
 
 impl Debug for App {
@@ -88,12 +86,24 @@ impl Debug for App {
 }
 
 /// Each `SubApp` has its own [`Schedule`] and [`World`], enabling a separation of concerns.
-struct SubApp {
+pub struct SubApp {
     app: App,
     extract: Box<dyn Fn(&mut World, &mut App) + Send + Sync>, // Send + Sync bound is only required to make SubApp send sync
     runner: Box<dyn Fn(&mut App) + Send + Sync>, // this send sync bound is required since we're actually sending this function to another thread
 }
 
+impl SubApp {
+    /// runs the `SubApp` with its runner
+    pub fn run(&mut self) {
+        (self.runner)(&mut self.app);
+    }
+
+    /// extract data from main world to sub app
+    pub fn extract(&mut self, main_world: &mut World) {
+        (self.extract)(main_world, &mut self.app);
+    }
+}
+
 impl Debug for SubApp {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "SubApp {{ app: ")?;
@@ -141,7 +151,6 @@ impl App {
             sub_apps: HashMap::default(),
             plugin_registry: Vec::default(),
             plugin_name_added: Default::default(),
-            run_once: false,
         }
     }
 
@@ -153,22 +162,10 @@ impl App {
     pub fn update(&mut self) {
         #[cfg(feature = "trace")]
         let _bevy_frame_update_span = info_span!("frame").entered();
-        let thread_executor = self
-            .world
-            .get_resource::<MainThreadExecutor>()
-            .map(|e| e.0.clone());
-        ComputeTaskPool::init(TaskPool::default).scope(thread_executor, |scope| {
-            if self.run_once {
-                for sub_app in self.sub_apps.values_mut() {
-                    scope.spawn(async { (sub_app.runner)(&mut sub_app.app) });
-                }
-            }
-            self.schedule.run(&mut self.world);
-        });
+        self.schedule.run(&mut self.world);
         for sub_app in self.sub_apps.values_mut() {
             (sub_app.extract)(&mut self.world, &mut sub_app.app);
         }
-        self.run_once = true;
     }
 
     /// Starts the application by calling the app's [runner function](Self::set_runner).
@@ -1057,6 +1054,16 @@ impl App {
         }
     }
 
+    /// inserts an existing sub app into the app
+    pub fn insert_sub_app(&mut self, label: impl AppLabel, sub_app: SubApp) {
+        self.sub_apps.insert(label.as_label(), sub_app);
+    }
+
+    /// remove a sub app from the app
+    pub fn remove_sub_app(&mut self, label: impl AppLabel) -> Option<SubApp> {
+        self.sub_apps.remove(&label.as_label())
+    }
+
     /// Retrieves a `SubApp` inside this [`App`] with the given label, if it exists. Otherwise returns
     /// an [`Err`] containing the given label.
     pub fn get_sub_app(&self, label: impl AppLabel) -> Result<&App, impl AppLabel> {

crates/bevy_internal/Cargo.toml

@@ -16,7 +16,8 @@ trace = [
     "bevy_ecs/trace",
     "bevy_log/trace",
     "bevy_render?/trace",
-    "bevy_hierarchy/trace"
+    "bevy_hierarchy/trace",
+    "bevy_winit/trace"
 ]
 trace_chrome = [ "bevy_log/tracing-chrome" ]
 trace_tracy = ["bevy_render?/tracing-tracy", "bevy_log/tracing-tracy" ]

crates/bevy_render/Cargo.toml

@@ -40,6 +40,7 @@ bevy_math = { path = "../bevy_math", version = "0.9.0-dev" }
 bevy_mikktspace = { path = "../bevy_mikktspace", version = "0.9.0-dev" }
 bevy_reflect = { path = "../bevy_reflect", version = "0.9.0-dev", features = ["bevy"] }
 bevy_render_macros = { path = "macros", version = "0.9.0-dev" }
+bevy_tasks = { path = "../bevy_tasks", version = "0.9.0-dev" }
 bevy_time = { path = "../bevy_time", version = "0.9.0-dev" }
 bevy_transform = { path = "../bevy_transform", version = "0.9.0-dev" }
 bevy_window = { path = "../bevy_window", version = "0.9.0-dev" }
@@ -75,3 +76,4 @@ basis-universal = { version = "0.2.0", optional = true }
 encase = { version = "0.4", features = ["glam"] }
 # For wgpu profiling using tracing. Use `RUST_LOG=info` to also capture the wgpu spans.
 profiling = { version = "1", features = ["profile-with-tracing"], optional = true }
+async-channel = "1.4"

crates/bevy_render/src/lib.rs

@@ -7,6 +7,7 @@ mod extract_param;
 pub mod extract_resource;
 pub mod globals;
 pub mod mesh;
+pub mod pipelined_rendering;
 pub mod primitives;
 pub mod rangefinder;
 pub mod render_asset;

crates/bevy_render/src/pipelined_rendering.rs

@@ -0,0 +1,85 @@
+use async_channel::{Receiver, Sender};
+use bevy_app::{App, SubApp};
+use bevy_ecs::{schedule::MainThreadExecutor, system::Resource, world::Mut};
+use bevy_tasks::ComputeTaskPool;
+
+#[cfg(feature = "trace")]
+use bevy_utils::tracing::Instrument;
+
+use crate::RenderApp;
+
+/// Resource to be used for pipelined rendering for sending the render app from the main thread to the rendering thread
+#[derive(Resource)]
+pub struct MainToRenderAppSender(pub Sender<SubApp>);
+
+/// Resource used by pipelined rendering to send the render app from the render thread to the main thread
+#[derive(Resource)]
+pub struct RenderToMainAppReceiver(pub Receiver<SubApp>);
+
+/// sets up the render thread and insert resource into the main app for controlling the render thread
+pub fn setup_pipelined_rendering(app: &mut App) {
+    let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::<SubApp>(1);
+    let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::<SubApp>(1);
+
+    let render_app = app.remove_sub_app(RenderApp).unwrap();
+    render_to_app_sender.send_blocking(render_app).unwrap();
+
+    app.insert_resource(MainToRenderAppSender(app_to_render_sender));
+    app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver));
+
+    let render_task = async move {
+        loop {
+            // TODO: exit loop when app is exited
+            let recv_task = app_to_render_receiver.recv();
+            #[cfg(feature = "trace")]
+            let span = bevy_utils::tracing::info_span!("receive render world from main");
+            #[cfg(feature = "trace")]
+            let recv_task = recv_task.instrument(span);
+            let mut sub_app = recv_task.await.unwrap();
+            sub_app.run();
+            render_to_app_sender.send(sub_app).await.unwrap();
+        }
+    };
+    #[cfg(feature = "trace")]
+    let span = bevy_utils::tracing::info_span!("render task");
+    #[cfg(feature = "trace")]
+    let render_task = render_task.instrument(span);
+    ComputeTaskPool::get().spawn(render_task).detach();
+}
+
+pub fn update_rendering(app: &mut App) {
+    app.update();
+
+    // wait to get the render app back to signal that rendering is finished
+    let mut render_app = app
+        .world
+        .resource_scope(|world, main_thread_executor: Mut<MainThreadExecutor>| {
+            ComputeTaskPool::get()
+                .scope(Some(main_thread_executor.0.clone()), |s| {
+                    s.spawn(async {
+                        let receiver = world.get_resource::<RenderToMainAppReceiver>().unwrap();
+                        let recv = receiver.0.recv();
+                        #[cfg(feature = "trace")]
+                        let span = bevy_utils::tracing::info_span!("wait for render");
+                        #[cfg(feature = "trace")]
+                        let recv = recv.instrument(span);
+                        recv.await.unwrap()
+                    });
+                })
+                .pop()
+        })
+        .unwrap();
+
+    render_app.extract(&mut app.world);
+
+    {
+        #[cfg(feature = "trace")]
+        let _span = bevy_utils::tracing::info_span!("send world to render").entered();
+        app.world
+            .resource_scope(|_world, sender: Mut<MainToRenderAppSender>| {
+                sender.0.send_blocking(render_app).unwrap();
+            });
+    }
+
+    // frame pacing plugin should run here somehow. i.e. after rendering, but before input handling
+}

crates/bevy_winit/Cargo.toml

@@ -9,6 +9,7 @@ license = "MIT OR Apache-2.0"
 keywords = ["bevy"]
 
 [features]
+trace = []
 wayland = ["winit/wayland"]
 x11 = ["winit/x11"]
 
@@ -18,6 +19,7 @@ bevy_app = { path = "../bevy_app", version = "0.9.0-dev" }
 bevy_ecs = { path = "../bevy_ecs", version = "0.9.0-dev" }
 bevy_input = { path = "../bevy_input", version = "0.9.0-dev" }
 bevy_math = { path = "../bevy_math", version = "0.9.0-dev" }
+bevy_render = { path = "../bevy_render", version = "0.9.0-dev" }
 bevy_window = { path = "../bevy_window", version = "0.9.0-dev" }
 bevy_utils = { path = "../bevy_utils", version = "0.9.0-dev" }
 

crates/bevy_winit/src/lib.rs

@@ -4,6 +4,7 @@ mod web_resize;
 mod winit_config;
 mod winit_windows;
 
+use bevy_render::pipelined_rendering::{setup_pipelined_rendering, update_rendering};
 use converters::convert_cursor_grab_mode;
 pub use winit_config::*;
 pub use winit_windows::*;
@@ -348,11 +349,15 @@ pub fn winit_runner_with(mut app: App) {
 
     let return_from_run = app.world.resource::<WinitSettings>().return_from_run;
 
+    setup_pipelined_rendering(&mut app);
+
     trace!("Entering winit event loop");
 
     let event_handler = move |event: Event<()>,
                               event_loop: &EventLoopWindowTarget<()>,
                               control_flow: &mut ControlFlow| {
+        #[cfg(feature = "trace")]
+        let _span = bevy_utils::tracing::info_span!("winit event_handler").entered();
         match event {
             event::Event::NewEvents(start) => {
                 let winit_config = app.world.resource::<WinitSettings>();
@@ -587,7 +592,8 @@ pub fn winit_runner_with(mut app: App) {
                 };
                 if update {
                     winit_state.last_update = Instant::now();
-                    app.update();
+                    update_rendering(&mut app);
+                    // app.update();
                 }
             }
             Event::RedrawEventsCleared => {