Skip to content

Commit eb3e236

Browse files
gatesngatesn
authored
Move Arrow dtype conversion into vortex-dtype (#2588)
Also bring datetime extension types in
1 parent 66efa9c commit eb3e236

File tree

47 files changed

+416
-442
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+416
-442
lines changed

‎Cargo.lock

+3-17
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ members = [
99
"vortex-btrblocks",
1010
"vortex-buffer",
1111
"vortex-datafusion",
12-
"vortex-datetime-dtype",
1312
"vortex-dtype",
1413
"vortex-error",
1514
"vortex-expr",
@@ -171,7 +170,6 @@ vortex-btrblocks = { version = "0.25.2", path = "./vortex-btrblocks" }
171170
vortex-buffer = { version = "0.25.2", path = "./vortex-buffer" }
172171
vortex-bytebool = { version = "0.25.2", path = "./encodings/bytebool" }
173172
vortex-datafusion = { version = "0.25.2", path = "./vortex-datafusion" }
174-
vortex-datetime-dtype = { version = "0.25.2", path = "./vortex-datetime-dtype" }
175173
vortex-datetime-parts = { version = "0.25.2", path = "./encodings/datetime-parts" }
176174
vortex-dict = { version = "0.25.2", path = "./encodings/dict" }
177175
vortex-dtype = { version = "0.25.2", path = "./vortex-dtype", default-features = false }

‎bench-vortex/src/bin/notimplemented.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use vortex::arrays::{
1010
VarBinViewArray,
1111
};
1212
use vortex::buffer::buffer;
13-
use vortex::datetime_dtype::{TIME_ID, TemporalMetadata, TimeUnit};
13+
use vortex::dtype::datetime::{TIME_ID, TemporalMetadata, TimeUnit};
1414
use vortex::dtype::{DType, ExtDType, Nullability, PType};
1515
use vortex::encodings::alp::{ALPArray, Exponents, RDEncoder};
1616
use vortex::encodings::bytebool::ByteBoolArray;

‎bench-vortex/src/clickbench.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ use futures::{StreamExt, TryStreamExt, stream};
1111
use tokio::fs::{OpenOptions, create_dir_all};
1212
use tracing::info;
1313
use vortex::TryIntoArray;
14-
use vortex::arrow::FromArrowType;
1514
use vortex::dtype::DType;
15+
use vortex::dtype::arrow::FromArrowType;
1616
use vortex::error::{VortexError, vortex_err};
1717
use vortex::file::{DEFAULT_REGISTRY, VORTEX_FILE_EXTENSION, VortexWriteOptions};
1818
use vortex::layout::{LayoutRegistry, LayoutRegistryExt};

‎bench-vortex/src/parquet_reader.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ use std::path::PathBuf;
44
use arrow_array::RecordBatchReader;
55
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
66
use vortex::TryIntoArray;
7-
use vortex::arrow::FromArrowType;
87
use vortex::dtype::DType;
8+
use vortex::dtype::arrow::FromArrowType;
99
use vortex::error::{VortexError, VortexResult};
1010
use vortex::iter::{ArrayIteratorAdapter, ArrayIteratorExt};
1111
use vortex::stream::ArrayStream;

‎bench-vortex/src/tpch/mod.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use object_store::path::Path as ObjectStorePath;
2121
use tokio::fs::OpenOptions;
2222
use url::Url;
2323
use vortex::arrays::ChunkedArray;
24-
use vortex::arrow::{FromArrowArray, FromArrowType};
24+
use vortex::arrow::FromArrowArray;
2525
use vortex::dtype::DType;
2626
use vortex::error::VortexExpect as _;
2727
use vortex::file::{DEFAULT_REGISTRY, VORTEX_FILE_EXTENSION, VortexWriteOptions};
@@ -38,6 +38,7 @@ mod execute;
3838
pub mod schema;
3939

4040
pub use execute::*;
41+
use vortex::dtype::arrow::FromArrowType;
4142
use vortex::error::VortexError;
4243
use vortex::stream::ArrayStreamAdapter;
4344

‎encodings/datetime-parts/Cargo.toml

-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ getrandom_v02 = { workspace = true }
2121
rkyv = { workspace = true }
2222
vortex-array = { workspace = true }
2323
vortex-buffer = { workspace = true }
24-
vortex-datetime-dtype = { workspace = true }
2524
vortex-dtype = { workspace = true }
2625
vortex-error = { workspace = true }
2726
vortex-mask = { workspace = true }

‎encodings/datetime-parts/src/canonical.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ use vortex_array::compute::try_cast;
33
use vortex_array::validity::Validity;
44
use vortex_array::{Array, ArrayCanonicalImpl, Canonical, ToCanonical};
55
use vortex_buffer::BufferMut;
6-
use vortex_datetime_dtype::{TemporalMetadata, TimeUnit};
76
use vortex_dtype::Nullability::NonNullable;
7+
use vortex_dtype::datetime::{TemporalMetadata, TimeUnit};
88
use vortex_dtype::{DType, PType};
99
use vortex_error::{VortexExpect as _, VortexResult, vortex_bail};
1010
use vortex_scalar::PrimitiveScalar;
@@ -103,7 +103,7 @@ mod test {
103103
use vortex_array::validity::Validity;
104104
use vortex_array::{Array, ToCanonical};
105105
use vortex_buffer::buffer;
106-
use vortex_datetime_dtype::TimeUnit;
106+
use vortex_dtype::datetime::TimeUnit;
107107

108108
use crate::DateTimePartsArray;
109109
use crate::canonical::decode_to_temporal;

‎encodings/datetime-parts/src/compress.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ mod tests {
6868
use vortex_array::validity::Validity;
6969
use vortex_array::{Array, ToCanonical};
7070
use vortex_buffer::buffer;
71-
use vortex_datetime_dtype::TimeUnit;
71+
use vortex_dtype::datetime::TimeUnit;
7272

7373
use crate::{TemporalParts, split_temporal};
7474

‎encodings/datetime-parts/src/compute/cast.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ mod tests {
3232
use vortex_array::validity::Validity;
3333
use vortex_array::{Array, ArrayRef};
3434
use vortex_buffer::buffer;
35-
use vortex_datetime_dtype::TimeUnit;
35+
use vortex_dtype::datetime::TimeUnit;
3636
use vortex_dtype::{DType, Nullability};
3737

3838
use crate::DateTimePartsArray;

‎encodings/datetime-parts/src/compute/compare.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
use vortex_array::arrays::ConstantArray;
22
use vortex_array::compute::{CompareFn, Operator, and, compare, or, try_cast};
33
use vortex_array::{Array, ArrayRef};
4-
use vortex_datetime_dtype::TemporalMetadata;
54
use vortex_dtype::DType;
5+
use vortex_dtype::datetime::TemporalMetadata;
66
use vortex_error::{VortexExpect as _, VortexResult};
77

88
use crate::array::{DateTimePartsArray, DateTimePartsEncoding};
@@ -157,8 +157,8 @@ mod test {
157157
use vortex_array::compute::Operator;
158158
use vortex_array::validity::Validity;
159159
use vortex_buffer::buffer;
160-
use vortex_datetime_dtype::TimeUnit;
161160
use vortex_dtype::NativePType;
161+
use vortex_dtype::datetime::TimeUnit;
162162

163163
use super::*;
164164

‎encodings/datetime-parts/src/compute/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ use vortex_array::compute::{
88
};
99
use vortex_array::vtable::ComputeVTable;
1010
use vortex_array::{Array, ArrayRef};
11-
use vortex_datetime_dtype::TemporalMetadata;
1211
use vortex_dtype::Nullability::{NonNullable, Nullable};
12+
use vortex_dtype::datetime::TemporalMetadata;
1313
use vortex_dtype::{DType, PType};
1414
use vortex_error::{VortexResult, vortex_bail};
1515
use vortex_scalar::Scalar;

‎encodings/datetime-parts/src/timestamp.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use vortex_datetime_dtype::TimeUnit;
1+
use vortex_dtype::datetime::TimeUnit;
22
use vortex_error::{VortexResult, vortex_bail};
33

44
pub const SECONDS_PER_DAY: i64 = 86_400; // 24 * 60 * 60

‎pyvortex/src/arrays/from_arrow.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ use arrow::record_batch::RecordBatchReader;
66
use pyo3::exceptions::PyValueError;
77
use pyo3::prelude::*;
88
use vortex::arrays::ChunkedArray;
9-
use vortex::arrow::{FromArrowArray, FromArrowType};
9+
use vortex::arrow::FromArrowArray;
1010
use vortex::dtype::DType;
11+
use vortex::dtype::arrow::FromArrowType;
1112
use vortex::error::{VortexError, VortexResult};
1213
use vortex::{Array, ArrayRef, TryIntoArray};
1314

‎pyvortex/src/arrays/mod.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use pyo3::exceptions::{PyTypeError, PyValueError};
1414
use pyo3::prelude::*;
1515
use pyo3::types::{PyDict, PyList};
1616
use vortex::arrays::ChunkedArray;
17-
use vortex::arrow::{IntoArrowArray, infer_data_type};
17+
use vortex::arrow::IntoArrowArray;
1818
use vortex::compute::{Operator, compare, fill_forward, scalar_at, slice, take};
1919
use vortex::error::VortexError;
2020
use vortex::mask::Mask;
@@ -243,7 +243,7 @@ impl PyArray {
243243
if let Some(chunked_array) = array.as_opt::<ChunkedArray>() {
244244
// We figure out a single Arrow Data Type to convert all chunks into, otherwise
245245
// the preferred type of each chunk may be different.
246-
let arrow_dtype = infer_data_type(chunked_array.dtype())?;
246+
let arrow_dtype = chunked_array.dtype().to_arrow_dtype()?;
247247

248248
let chunks = chunked_array
249249
.chunks()

‎pyvortex/src/dataset.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ use pyo3::exceptions::PyTypeError;
88
use pyo3::prelude::*;
99
use pyo3::types::PyString;
1010
use vortex::arrays::ChunkedArray;
11-
use vortex::arrow::infer_schema;
1211
use vortex::dtype::FieldName;
1312
use vortex::error::VortexResult;
1413
use vortex::expr::{ExprRef, Select, ident};
@@ -101,7 +100,7 @@ impl TokioFileDataset {
101100
pub async fn try_new(path: String) -> VortexResult<Self> {
102101
let file = TokioFile::open(path)?;
103102
let vxf = VortexOpenOptions::file(file).open().await?;
104-
let schema = Arc::new(infer_schema(vxf.dtype())?);
103+
let schema = Arc::new(vxf.dtype().to_arrow_schema()?);
105104

106105
Ok(Self { vxf, schema })
107106
}
@@ -189,7 +188,7 @@ impl ObjectStoreUrlDataset {
189188
let reader = vortex_read_at_from_url(&url).await?;
190189

191190
let vxf = VortexOpenOptions::file(reader).open().await?;
192-
let schema = Arc::new(infer_schema(vxf.dtype())?);
191+
let schema = Arc::new(vxf.dtype().to_arrow_schema()?);
193192

194193
Ok(Self { vxf, schema })
195194
}

‎pyvortex/src/dtype/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ use pyo3::{
1818
Bound, PyAny, PyClass, PyClassInitializer, PyResult, Python, pyclass, pymethods,
1919
wrap_pyfunction,
2020
};
21-
use vortex::arrow::FromArrowType;
2221
use vortex::dtype::DType;
22+
use vortex::dtype::arrow::FromArrowType;
2323

2424
use crate::dtype::binary::PyBinaryDType;
2525
use crate::dtype::bool::PyBoolDType;

‎pyvortex/src/record_batch_reader.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ use arrow::datatypes::SchemaRef;
77
use arrow::error::ArrowError;
88
use futures::StreamExt;
99
use vortex::ArrayRef;
10-
use vortex::arrow::infer_schema;
1110
use vortex::error::{VortexError, VortexResult};
1211
use vortex::stream::ArrayStream;
1312

@@ -43,7 +42,7 @@ where
4342
AR: AsyncRuntime,
4443
{
4544
pub fn try_new(stream: S, runtime: &'a AR) -> VortexResult<Self> {
46-
let arrow_schema = Arc::new(infer_schema(stream.dtype())?);
45+
let arrow_schema = Arc::new(stream.dtype().to_arrow_schema()?);
4746
let stream = Box::pin(stream);
4847
Ok(VortexRecordBatchReader {
4948
stream,

‎vortex-array/Cargo.toml

+1-2
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,7 @@ rkyv = { workspace = true }
5151
serde = { workspace = true, features = ["derive"] }
5252
static_assertions = { workspace = true }
5353
vortex-buffer = { workspace = true, features = ["arrow", "rkyv"] }
54-
vortex-datetime-dtype = { workspace = true }
55-
vortex-dtype = { workspace = true, features = ["rkyv", "serde"] }
54+
vortex-dtype = { workspace = true, features = ["arrow", "rkyv", "serde"] }
5655
vortex-error = { workspace = true, features = [
5756
"flatbuffers",
5857
"flexbuffers",

‎vortex-array/src/arrays/datetime/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ mod test;
33

44
use std::sync::Arc;
55

6-
use vortex_datetime_dtype::{DATE_ID, TIME_ID, TIMESTAMP_ID, TemporalMetadata, TimeUnit};
6+
use vortex_dtype::datetime::{DATE_ID, TIME_ID, TIMESTAMP_ID, TemporalMetadata, TimeUnit};
77
use vortex_dtype::{DType, ExtDType};
88
use vortex_error::{VortexError, vortex_err, vortex_panic};
99

‎vortex-array/src/arrays/datetime/test.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use rstest::rstest;
22
use vortex_buffer::buffer;
3-
use vortex_datetime_dtype::{TemporalMetadata, TimeUnit};
3+
use vortex_dtype::datetime::{TemporalMetadata, TimeUnit};
44

55
use crate::array::Array;
66
use crate::arrays::{PrimitiveArray, TemporalArray};

‎vortex-array/src/arrays/extension/compute/to_arrow.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use arrow_array::{
66
TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
77
};
88
use arrow_schema::DataType;
9-
use vortex_datetime_dtype::{TemporalMetadata, TimeUnit, is_temporal_ext_type};
9+
use vortex_dtype::datetime::{TemporalMetadata, TimeUnit, is_temporal_ext_type};
1010
use vortex_dtype::{DType, NativePType};
1111
use vortex_error::{VortexResult, vortex_bail};
1212

‎vortex-array/src/arrow/array.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
1717
use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
1818
use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
1919
use vortex_buffer::{Alignment, Buffer, ByteBuffer};
20-
use vortex_datetime_dtype::TimeUnit;
20+
use vortex_dtype::datetime::TimeUnit;
2121
use vortex_dtype::{DType, NativePType, PType};
2222
use vortex_error::{VortexExpect as _, vortex_panic};
2323

0 commit comments

Comments
 (0)