Generic Arrow Flight RPC data source with a Flight SQL adapter

apache · Aug 12, 2024 · c38db1c · c38db1c
1 parent 64a9280
commit c38db1c
Show file tree

Hide file tree

Showing 9 changed files with 922 additions and 1 deletion.
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -44,6 +44,7 @@ datafusion = { path = "../datafusion/core", version = "41.0.0", features = [
     "regex_expressions",
     "unicode_expressions",
     "compression",
+    "flight",
 ] }
 dirs = "4.0.0"
 env_logger = "0.9"

diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -59,10 +59,12 @@ default = [
     "unicode_expressions",
     "compression",
     "parquet",
+    "flight",
 ]
 encoding_expressions = ["datafusion-functions/encoding_expressions"]
 # Used for testing ONLY: causes all values to hash to the same value (test for collisions)
 force_hash_collisions = ["datafusion-physical-plan/force_hash_collisions", "datafusion-common/force_hash_collisions"]
+flight = ["dep:arrow-flight", "dep:tonic"]
 math_expressions = ["datafusion-functions/math_expressions"]
 parquet = ["datafusion-common/parquet", "dep:parquet"]
 pyarrow = ["datafusion-common/pyarrow", "parquet"]
@@ -83,6 +85,7 @@ ahash = { workspace = true }
 apache-avro = { version = "0.16", optional = true }
 arrow = { workspace = true }
 arrow-array = { workspace = true }
+arrow-flight = { workspace = true, optional = true }
 arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
 async-compression = { version = "0.4.0", features = [
@@ -133,6 +136,7 @@ sqlparser = { workspace = true }
 tempfile = { workspace = true }
 tokio = { workspace = true }
 tokio-util = { version = "0.7.4", features = ["io"], optional = true }
+tonic = { version = "0.11", optional = true }
 url = { workspace = true }
 uuid = { version = "1.7", features = ["v4"] }
 xz2 = { version = "0.1", optional = true, features = ["static"] }
@@ -151,6 +155,7 @@ half = { workspace = true, default-features = true }
 paste = "^1.0"
 postgres-protocol = "0.6.4"
 postgres-types = { version = "0.2.4", features = ["derive", "with-chrono-0_4"] }
+prost = "0.12"
 rand = { workspace = true, features = ["small_rng"] }
 rand_distr = "0.4.3"
 regex = { workspace = true }
@@ -161,6 +166,7 @@ test-utils = { path = "../../test-utils" }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }
 tokio-postgres = "0.7.7"
+tokio-stream = { version = "0.1.15", features = ["net"] }
 
 [target.'cfg(not(target_os = "windows"))'.dev-dependencies]
 nix = { version = "0.29.0", features = ["fs"] }