Add e2e ci test

Author: xinlifoobar

datafusion-cli/Cargo.toml

@@ -65,4 +65,4 @@ url = "2.2"
 assert_cmd = "2.0"
 ctor = "0.2.0"
 predicates = "3.0"
-rstest = "0.17"
+rstest = "0.17"

datafusion-cli/src/hf_store.rs

@@ -607,7 +607,7 @@ impl ObjectStore for HFStore {
         let file_path_prefix = parsed_url.file_path_prefix();
 
         futures::stream::once(async move {
-            let result = self.store.get(&Path::from(tree_path)).await?;
+            let result = self.store.get(&Path::parse(tree_path)?).await?;
             let Ok(bytes) = result.bytes().await else {
                 return Err(ObjectStoreError::Generic {
                     store: STORE,
@@ -630,7 +630,7 @@ impl ObjectStore for HFStore {
                     .filter(|entry| entry.is_file())
                     .map(|entry| format!("{}/{}", file_path_prefix, entry.path.clone()))
                     .map(|meta_location| async {
-                        self.store.head(&Path::from(meta_location)).await
+                        self.store.head(&Path::parse(meta_location)?).await
                     }),
             )
             .await
@@ -646,9 +646,10 @@ impl ObjectStore for HFStore {
                                 .into(),
                         });
                     };
-                    meta.location = Path::from(location.hf_path());
+
+                    meta.location = Path::from_url_path(location.hf_path())?;
                     if let Some(e_tag) = meta.e_tag.as_deref() {
-                        meta.e_tag = Some(e_tag.replace("\"", ""));
+                        meta.e_tag = Some(e_tag.replace('"', ""));
                     }
 
                     Ok(meta)

datafusion-cli/tests/cli_integration.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::fs;
 use std::process::Command;
 
 use assert_cmd::prelude::{CommandCargoExt, OutputAssertExt};
@@ -54,3 +55,35 @@ fn cli_quick_test<'a>(
     cmd.args(args);
     cmd.assert().stdout(predicate::eq(expected));
 }
+
+#[rstest]
+#[case::exec_hf_store_test(
+    ["--file", "tests/data/hf_store_sql.txt", "--format", "json", "-q"],
+    "tests/data/hf_store_expected.jsonl",
+)]
+#[test]
+fn cli_hf_store_test<'a>(
+    #[case] args: impl IntoIterator<Item = &'a str>,
+    #[case] expected_file: &str,
+) {
+    let mut cmd = Command::cargo_bin("datafusion-cli").unwrap();
+    cmd.args(args);
+
+    let actual: Vec<serde_json::Value> = serde_json::Deserializer::from_str(
+        String::from_utf8(cmd.assert().get_output().stdout.to_vec())
+            .unwrap()
+            .as_str(),
+    )
+    .into_iter::<serde_json::Value>()
+    .collect::<Result<Vec<serde_json::Value>, _>>()
+    .unwrap();
+
+    let expected: Vec<serde_json::Value> = serde_json::Deserializer::from_str(
+        fs::read_to_string(expected_file).unwrap().as_str(),
+    )
+    .into_iter::<serde_json::Value>()
+    .collect::<Result<Vec<serde_json::Value>, _>>()
+    .unwrap();
+
+    assert_eq!(actual, expected);
+}

datafusion-cli/tests/data/hf_store_expected.jsonl

@@ -0,0 +1,20 @@
+[
+  {
+    "COUNT(*)": 5
+  }
+]
+[
+  {
+    "COUNT(*)": 152
+  }
+]
+[
+  {
+    "COUNT(*)": 173
+  }
+]
+[
+  {
+    "COUNT(*)": 152
+  }
+]

datafusion-cli/tests/data/hf_store_sql.txt

@@ -0,0 +1,9 @@
+select count(*) from "hf://datasets/cais/mmlu/astronomy/dev-00000-of-00001.parquet";
+
+select count(*) from "hf://datasets/cais/mmlu@~parquet/astronomy/test/0000.parquet";
+
+create external table test stored as parquet location "hf://datasets/cais/mmlu/astronomy/";
+SELECT count(*) FROM test;
+
+create external table test_revision stored as parquet location "hf://datasets/cais/mmlu@~parquet/astronomy/test/";
+SELECT count(*) FROM test_revision;