Feat: Implement hf:// / "hugging face" integration in datafusion-cli

datafusion-cli/Cargo.toml

@@ -34,6 +34,7 @@ arrow = "51.0.0"
 async-trait = "0.1.41"
 aws-config = "0.55"
 aws-credential-types = "0.55"
+bytes = "1.0"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "38.0.0", features = [
     "avro",
@@ -48,17 +49,20 @@ datafusion = { path = "../datafusion/core", version = "38.0.0", features = [
 dirs = "4.0.0"
 env_logger = "0.9"
 futures = "0.3"
+http= "0.2.12"
 mimalloc = { version = "0.1", default-features = false }
 object_store = { version = "0.9.0", features = ["aws", "gcp", "http"] }
 parking_lot = { version = "0.12" }
 parquet = { version = "51.0.0", default-features = false }
 regex = "1.8"
 rustyline = "11.0"
+serde = "1.0.117"
+serde_json = "1.0.117"
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }
 url = "2.2"
 
 [dev-dependencies]
 assert_cmd = "2.0"
 ctor = "0.2.0"
 predicates = "3.0"
-rstest = "0.17"
+rstest = "0.17"

datafusion-cli/src/catalog.rs

@@ -18,6 +18,7 @@
 use std::any::Any;
 use std::sync::{Arc, Weak};
 
+use crate::hf_store::HFOptions;
 use crate::object_storage::{get_object_store, AwsOptions, GcpOptions};
 
 use datafusion::catalog::schema::SchemaProvider;
@@ -183,6 +184,9 @@ impl SchemaProvider for DynamicFileSchemaProvider {
                     "gs" | "gcs" => {
                         state = state.add_table_options_extension(GcpOptions::default())
                     }
+                    "hf" => {
+                        state = state.add_table_options_extension(HFOptions::default());
+                    }
                     _ => {}
                 };
                 let store = get_object_store(