cleanup

Author: joseph-isaacs

Cargo.lock

@@ -22,14 +22,12 @@ crate-type = ["staticlib"]
 [dependencies]
 duckdb = { version = "1.2.0", features = ["vtab-loadable"] }
 duckdb-loadable-macros = "0.1.4"
+futures = { workspace = true, features = ["alloc"] }
 libduckdb-sys = { version = "1.2.0", features = ["loadable-extension"] }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
 vortex-array = { workspace = true }
-vortex-buffer = { workspace = true }
 vortex-duckdb = { workspace = true }
-vortex-dtype = { workspace = true }
-vortex-layout = { workspace = true, features = ["tokio"] }
 vortex-error = { workspace = true }
-futures = { workspace = true, features = ["alloc"] }
 vortex-file = { workspace = true, features = ["tokio"] }
 vortex-io = { workspace = true, features = ["tokio"] }
-tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
+vortex-layout = { workspace = true, features = ["tokio"] }

duckdb-vortex-rs/Cargo.toml

@@ -89,35 +89,13 @@ impl VTab for HelloVTab {
         let arr = rt.block_on(async {
             let mut stream = bind_data.stream.lock().await;
             stream.next().await
-
-            // let pos = init_data.position.load(Ordering::SeqCst);
-            // let next_pos = min(pos + 2048, arr.len());
-            // let arr = slice(arr, pos, next_pos).unwrap();
-            // init_data.position.store(next_pos, Ordering::SeqCst);
         });
 
         let Some(arr) = arr else {
             output.set_len(0);
             return Ok(());
         };
 
-        //             Ok(vxf
-        //                 .scan()
-        //                 .with_projection(projection)
-        //                 .with_some_filter(filter)
-        //                 .with_prefetch_conjuncts(true)
-        //                 .with_canonicalize(true)
-        //                 // DataFusion likes ~8k row batches. Ideally we would respect the config,
-        //                 // but at the moment our scanner has too much overhead to process small
-        //                 // batches efficiently.
-        //                 .with_split_by(SplitBy::RowCount(8 * batch_size))
-        //                 .with_task_executor(executor)
-        //                 .into_array_stream()?
-        //                 .map(move |array| {
-        //                     let st = array?.to_struct()?;
-        //                     Ok(st.into_record_batch_with_schema(projected_arrow_schema.as_ref())?)
-        //                 })
-        //                 .boxed())
         let arr = arr.unwrap();
         let struct_a = arr.to_struct().unwrap();
         let _null = to_duckdb_chunk(&struct_a, output).unwrap();

duckdb-vortex-rs/src/lib.rs

@@ -21,14 +21,12 @@ pub trait ToDuckDB {
 pub fn to_duckdb_chunk(
     struct_array: &StructArray,
     chunk: &mut DataChunkHandle,
-) -> VortexResult<Vec<bool>> {
-    let mut nullable = vec![false; struct_array.len()];
+) -> VortexResult<()> {
     for (idx, field) in struct_array.fields().iter().enumerate() {
         field.to_duckdb(&mut DataChunkHandleSlice::new(chunk, idx))?;
-        nullable[idx] = field.dtype().is_nullable();
     }
     chunk.set_len(struct_array.len());
-    Ok(nullable)
+    Ok(())
 }
 
 impl ToDuckDB for ArrayRef {
@@ -90,7 +88,6 @@ impl FromDuckDB<SizedFlatVector> for ArrayRef {
 #[cfg(test)]
 mod tests {
     use duckdb::core::DataChunkHandle;
-    use itertools::Itertools;
     use vortex_array::arrays::{BoolArray, PrimitiveArray, StructArray, VarBinArray};
     use vortex_array::validity::Validity;
     use vortex_array::variants::StructArrayTrait;
@@ -122,16 +119,16 @@ mod tests {
     #[test]
     fn test_vortex_to_duckdb() {
         let arr = data();
-        let ddb_type = arr
+        let (nullable, ddb_type): (Vec<_>, Vec<_>) = arr
             .dtype()
             .as_struct()
             .unwrap()
             .fields()
-            .map(|f| f.to_duckdb_type().unwrap())
-            .collect_vec();
+            .map(|f| (f.is_nullable(), f.to_duckdb_type().unwrap()))
+            .unzip();
         let struct_arr = arr.to_struct().unwrap();
         let mut output_chunk = DataChunkHandle::new(ddb_type.as_slice());
-        let nullable = to_duckdb_chunk(&struct_arr, &mut output_chunk).unwrap();
+        to_duckdb_chunk(&struct_arr, &mut output_chunk).unwrap();
 
         let vx_arr = ArrayRef::from_duckdb(&NamedDataChunk::new(
             &output_chunk,