From d9449e5ea0cc13afdf71118216a2141188566bec Mon Sep 17 00:00:00 2001 From: hozan23 Date: Fri, 23 Aug 2024 10:59:20 +0200 Subject: [PATCH] "create datafusion-flight-sql-table-provider crate & move the executor from datafusion-flight-sql-server" --- Cargo.toml | 1 + README.md | 3 +++ datafusion-flight-sql-server/Cargo.toml | 5 +++- .../examples/flight-sql.rs | 13 ++++------ .../examples/test.csv | 4 ++++ datafusion-flight-sql-server/src/lib.rs | 5 ++-- .../src/server/mod.rs | 6 ----- .../src/{server => }/service.rs | 5 ++-- .../src/{server => }/session.rs | 0 .../src/{server => }/state.rs | 0 .../Cargo.toml | 24 +++++++++++++++++++ .../src/lib.rs | 0 12 files changed, 46 insertions(+), 20 deletions(-) rename {examples => datafusion-flight-sql-server}/examples/flight-sql.rs (88%) create mode 100644 datafusion-flight-sql-server/examples/test.csv delete mode 100644 datafusion-flight-sql-server/src/server/mod.rs rename datafusion-flight-sql-server/src/{server => }/service.rs (99%) rename datafusion-flight-sql-server/src/{server => }/session.rs (100%) rename datafusion-flight-sql-server/src/{server => }/state.rs (100%) create mode 100644 datafusion-flight-sql-table-provider/Cargo.toml rename datafusion-flight-sql-server/src/executor/mod.rs => datafusion-flight-sql-table-provider/src/lib.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 53ee52c..83ca62e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ resolver = "2" members = [ "datafusion-federation", "datafusion-flight-sql-server", + "datafusion-flight-sql-table-provider", ] [workspace.package] diff --git a/README.md b/README.md index 028db87..ed7fdef 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ The goal of this repo is to allow [DataFusion](https://github.com/apache/arrow-datafusion) to resolve queries across remote query engines while pushing down as much compute as possible down. + +> :warning: **All the examples are deprecated for now** + Check out [the examples](./examples/) to get a feel for how it works. Potential use-cases: diff --git a/datafusion-flight-sql-server/Cargo.toml b/datafusion-flight-sql-server/Cargo.toml index 8e466e2..ede6979 100644 --- a/datafusion-flight-sql-server/Cargo.toml +++ b/datafusion-flight-sql-server/Cargo.toml @@ -9,7 +9,6 @@ name = "datafusion_flight_sql_server" path = "src/lib.rs" [dependencies] -async-trait.workspace = true datafusion.workspace = true datafusion-substrait.workspace = true # XXX use the release verion on crates.io @@ -28,3 +27,7 @@ prost = "0.12.3" arrow = "52.0.0" arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] } log = "0.4.22" + +[dev-dependencies] +tokio = { version = "1.39.3", features = ["full"] } +datafusion-flight-sql-table-provider = { path = "../datafusion-flight-sql-table-provider" } diff --git a/examples/examples/flight-sql.rs b/datafusion-flight-sql-server/examples/flight-sql.rs similarity index 88% rename from examples/examples/flight-sql.rs rename to datafusion-flight-sql-server/examples/flight-sql.rs index e0899cb..021227a 100644 --- a/examples/examples/flight-sql.rs +++ b/datafusion-flight-sql-server/examples/flight-sql.rs @@ -2,15 +2,16 @@ use std::{sync::Arc, time::Duration}; use arrow_flight::sql::client::FlightSqlServiceClient; use datafusion::{ - catalog::schema::SchemaProvider, + catalog::SchemaProvider, error::{DataFusionError, Result}, execution::{ context::{SessionContext, SessionState}, options::CsvReadOptions, }, }; -use datafusion_federation_flight_sql::{executor::FlightSQLExecutor, server::FlightSqlService}; -use datafusion_federation_sql::{SQLFederationProvider, SQLSchemaProvider}; +use datafusion_federation::sql::{SQLFederationProvider, SQLSchemaProvider}; +use datafusion_flight_sql_server::service::FlightSqlService; +use datafusion_flight_sql_table_provider::FlightSQLExecutor; use tokio::time::sleep; use tonic::transport::Endpoint; @@ -19,11 +20,7 @@ async fn main() -> Result<()> { let dsn: String = "0.0.0.0:50051".to_string(); let remote_ctx = SessionContext::new(); remote_ctx - .register_csv( - "test", - "./examples/examples/test.csv", - CsvReadOptions::new(), - ) + .register_csv("test", "./examples/test.csv", CsvReadOptions::new()) .await?; // Remote context diff --git a/datafusion-flight-sql-server/examples/test.csv b/datafusion-flight-sql-server/examples/test.csv new file mode 100644 index 0000000..811d276 --- /dev/null +++ b/datafusion-flight-sql-server/examples/test.csv @@ -0,0 +1,4 @@ +foo,bar +a,1 +b,2 +c,3 \ No newline at end of file diff --git a/datafusion-flight-sql-server/src/lib.rs b/datafusion-flight-sql-server/src/lib.rs index a8795c9..f7f6eac 100644 --- a/datafusion-flight-sql-server/src/lib.rs +++ b/datafusion-flight-sql-server/src/lib.rs @@ -1,2 +1,3 @@ -pub mod executor; -pub mod server; +pub mod service; +pub mod session; +pub mod state; diff --git a/datafusion-flight-sql-server/src/server/mod.rs b/datafusion-flight-sql-server/src/server/mod.rs deleted file mode 100644 index 0c054b9..0000000 --- a/datafusion-flight-sql-server/src/server/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod service; -pub use service::*; -mod state; -pub use state::*; -mod session; -pub use session::*; diff --git a/datafusion-flight-sql-server/src/server/service.rs b/datafusion-flight-sql-server/src/service.rs similarity index 99% rename from datafusion-flight-sql-server/src/server/service.rs rename to datafusion-flight-sql-server/src/service.rs index afa4a5f..8a85132 100644 --- a/datafusion-flight-sql-server/src/server/service.rs +++ b/datafusion-flight-sql-server/src/service.rs @@ -1,5 +1,4 @@ -use std::pin::Pin; -use std::sync::Arc; +use std::{pin::Pin, sync::Arc}; use arrow::{datatypes::SchemaRef, error::ArrowError, ipc::writer::IpcWriteOptions}; use arrow_flight::sql::{ @@ -40,8 +39,8 @@ use prost::bytes::Bytes; use tonic::transport::Server; use tonic::{Request, Response, Status, Streaming}; +use super::session::{SessionStateProvider, StaticSessionStateProvider}; use super::state::{CommandTicket, QueryHandle}; -use super::{SessionStateProvider, StaticSessionStateProvider}; type Result = std::result::Result; diff --git a/datafusion-flight-sql-server/src/server/session.rs b/datafusion-flight-sql-server/src/session.rs similarity index 100% rename from datafusion-flight-sql-server/src/server/session.rs rename to datafusion-flight-sql-server/src/session.rs diff --git a/datafusion-flight-sql-server/src/server/state.rs b/datafusion-flight-sql-server/src/state.rs similarity index 100% rename from datafusion-flight-sql-server/src/server/state.rs rename to datafusion-flight-sql-server/src/state.rs diff --git a/datafusion-flight-sql-table-provider/Cargo.toml b/datafusion-flight-sql-table-provider/Cargo.toml new file mode 100644 index 0000000..4302133 --- /dev/null +++ b/datafusion-flight-sql-table-provider/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "datafusion-flight-sql-table-provider" +version.workspace = true +edition.workspace = true +license.workspace = true +readme.workspace = true + +[dependencies] +async-trait.workspace = true +datafusion.workspace = true +# XXX use the release verion on crates.io +datafusion-federation = { path = "../datafusion-federation", features = [ + "sql", +] } + +futures = "0.3.30" +tonic = { version = "0.11.0", features = [ + "tls", + "transport", + "codegen", + "prost", +] } +arrow = "52.0.0" +arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] } diff --git a/datafusion-flight-sql-server/src/executor/mod.rs b/datafusion-flight-sql-table-provider/src/lib.rs similarity index 100% rename from datafusion-flight-sql-server/src/executor/mod.rs rename to datafusion-flight-sql-table-provider/src/lib.rs