feat: add fundamentals for new async-based streaming execution engine (…

…#16884)

Cargo.toml

@@ -31,6 +31,7 @@ arrow-data = { version = ">=41", default-features = false }
 arrow-schema = { version = ">=41", default-features = false }
 atoi = "2"
 atoi_simd = "0.15.5"
+atomic-waker = "1"
 avro-schema = { version = "0.3" }
 base64 = "0.22.0"
 bitflags = "2"
@@ -39,7 +40,9 @@ chrono = { version = "0.4.31", default-features = false, features = ["std"] }
 chrono-tz = "0.8.1"
 ciborium = "0.2"
 crossbeam-channel = "0.5.8"
+crossbeam-deque = "0.8.5"
 crossbeam-queue = "0.3"
+crossbeam-utils = "0.8.20"
 either = "1.11"
 ethnum = "1.3.2"
 fallible-streaming-iterator = "0.1.9"
@@ -57,7 +60,9 @@ ndarray = { version = "0.15", default-features = false }
 num-traits = "0.2"
 object_store = { version = "0.9", default-features = false }
 once_cell = "1"
+parking_lot = "0.12"
 percent-encoding = "2.3"
+pin-project-lite = "0.2"
 pyo3 = "0.21"
 rand = "0.8"
 rand_distr = "0.4"
@@ -71,6 +76,7 @@ serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1"
 simd-json = { version = "0.13", features = ["known-key"] }
 simdutf8 = "0.1.4"
+slotmap = "1"
 smartstring = "1"
 sqlparser = "0.45"
 stacker = "0.1"

crates/polars-stream/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "polars-stream"
+version = { workspace = true }
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+description = "Private crate for the streaming execution engine for the Polars DataFrame library"
+
+[dependencies]
+atomic-waker = { workspace = true }
+crossbeam-deque = { workspace = true }
+crossbeam-utils = { workspace = true }
+parking_lot = { workspace = true }
+pin-project-lite = { workspace = true }
+polars-utils = { workspace = true }
+rand = { workspace = true }
+slotmap = { workspace = true }
+
+[build-dependencies]
+version_check = { workspace = true }
+
+[features]
+nightly = []

crates/polars-stream/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

crates/polars-stream/README.md

@@ -0,0 +1,5 @@
+# polars-stream
+
+`polars-stream` is an **internal sub-crate** of the [Polars](https://crates.io/crates/polars) library, containing a streaming execution engine.
+
+**Important Note**: This crate is **not intended for external usage**. Please refer to the main [Polars crate](https://crates.io/crates/polars) for intended usage.

crates/polars-stream/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    let channel = version_check::Channel::read().unwrap();
+    if channel.is_nightly() {
+        println!("cargo:rustc-cfg=feature=\"nightly\"");
+    }
+}