add arrow-rs

Cargo.toml

@@ -82,6 +82,10 @@ polars-json = { version = "0.33.2", path = "crates/polars-json", default-feature
 polars = { version = "0.33.2", path = "crates/polars", default-features = false }
 rand_distr = "0.4"
 reqwest = { version = "0.11", default-features = false }
+arrow-array = {version = ">=41", default-features = false}
+arrow-buffer = { version = ">=41", default-features = false }
+arrow-data = { version = ">=41", default-features = false }
+arrow-schema = { version = ">=41", default-features = false }
 
 [workspace.dependencies.arrow]
 package = "nano-arrow"

crates/nano-arrow/Cargo.toml

@@ -68,10 +68,10 @@ multiversion = { workspace = true, optional = true }
 ahash = { workspace = true }
 
 # Support conversion to/from arrow-rs
-arrow-array = { version = ">=40", optional = true }
-arrow-buffer = { version = ">=40", optional = true }
-arrow-data = { version = ">=40", optional = true }
-arrow-schema = { version = ">=40", optional = true }
+arrow-array = { workspace = true, optional = true }
+arrow-buffer = { workspace = true, optional = true }
+arrow-data = { workspace = true, optional = true }
+arrow-schema = { workspace = true, optional = true }
 
 # parquet support
 [dependencies.parquet2]

crates/nano-arrow/src/array/binary/mod.rs

@@ -20,7 +20,7 @@ pub use mutable_values::*;
 mod mutable;
 pub use mutable::*;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 
 /// A [`BinaryArray`] is Arrow's semantically equivalent of an immutable `Vec<Option<Vec<u8>>>`.

crates/nano-arrow/src/array/boolean/mod.rs

@@ -7,7 +7,7 @@ use crate::datatypes::{DataType, PhysicalType};
 use crate::error::Error;
 use crate::trusted_len::TrustedLen;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/dictionary/mod.rs

@@ -9,7 +9,7 @@ use crate::scalar::{new_scalar, Scalar};
 use crate::trusted_len::TrustedLen;
 use crate::types::NativeType;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/fixed_size_binary/mod.rs

@@ -4,7 +4,7 @@ use crate::buffer::Buffer;
 use crate::datatypes::DataType;
 use crate::error::Error;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/fixed_size_list/mod.rs

@@ -3,7 +3,7 @@ use crate::bitmap::Bitmap;
 use crate::datatypes::{DataType, Field};
 use crate::error::Error;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/list/mod.rs

@@ -5,7 +5,7 @@ use crate::datatypes::{DataType, Field};
 use crate::error::Error;
 use crate::offset::{Offset, Offsets, OffsetsBuffer};
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/map/mod.rs

@@ -5,7 +5,7 @@ use crate::datatypes::{DataType, Field};
 use crate::error::Error;
 use crate::offset::OffsetsBuffer;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/mod.rs

@@ -399,7 +399,7 @@ pub fn new_null_array(data_type: DataType, length: usize) -> Box<dyn Array> {
 /// Trait providing bi-directional conversion between arrow2 [`Array`] and arrow-rs [`ArrayData`]
 ///
 /// [`ArrayData`]: arrow_data::ArrayData
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 pub trait Arrow2Arrow: Array {
     /// Convert this [`Array`] into [`ArrayData`]
     fn to_data(&self) -> arrow_data::ArrayData;
@@ -408,44 +408,44 @@ pub trait Arrow2Arrow: Array {
     fn from_data(data: &arrow_data::ArrayData) -> Self;
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 macro_rules! to_data_dyn {
     ($array:expr, $ty:ty) => {{
         let f = |x: &$ty| x.to_data();
         general_dyn!($array, $ty, f)
     }};
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<Box<dyn Array>> for arrow_array::ArrayRef {
     fn from(value: Box<dyn Array>) -> Self {
         value.as_ref().into()
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<&dyn Array> for arrow_array::ArrayRef {
     fn from(value: &dyn Array) -> Self {
         arrow_array::make_array(to_data(value))
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_array::ArrayRef> for Box<dyn Array> {
     fn from(value: arrow_array::ArrayRef) -> Self {
         value.as_ref().into()
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<&dyn arrow_array::Array> for Box<dyn Array> {
     fn from(value: &dyn arrow_array::Array) -> Self {
         from_data(&value.to_data())
     }
 }
 
 /// Convert an arrow2 [`Array`] to [`arrow_data::ArrayData`]
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 pub fn to_data(array: &dyn Array) -> arrow_data::ArrayData {
     use crate::datatypes::PhysicalType::*;
     match array.data_type().to_physical_type() {
@@ -474,7 +474,7 @@ pub fn to_data(array: &dyn Array) -> arrow_data::ArrayData {
 }
 
 /// Convert an [`arrow_data::ArrayData`] to arrow2 [`Array`]
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 pub fn from_data(data: &arrow_data::ArrayData) -> Box<dyn Array> {
     use crate::datatypes::PhysicalType::*;
     let data_type: DataType = data.data_type().clone().into();

crates/nano-arrow/src/array/null.rs

@@ -178,7 +178,7 @@ impl<A: ffi::ArrowArrayRef> FromFfi<A> for NullArray {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod arrow {
     use arrow_data::{ArrayData, ArrayDataBuilder};
 

crates/nano-arrow/src/array/primitive/mod.rs

@@ -9,7 +9,7 @@ use crate::error::Error;
 use crate::trusted_len::TrustedLen;
 use crate::types::{days_ms, f16, i256, months_days_ns, NativeType};
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/struct_/mod.rs

@@ -3,7 +3,7 @@ use crate::bitmap::Bitmap;
 use crate::datatypes::{DataType, Field};
 use crate::error::Error;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/union/mod.rs

@@ -5,7 +5,7 @@ use crate::datatypes::{DataType, Field, UnionMode};
 use crate::error::Error;
 use crate::scalar::{new_scalar, Scalar};
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/array/utf8/mod.rs

@@ -10,7 +10,7 @@ use crate::error::{Error, Result};
 use crate::offset::{Offset, Offsets, OffsetsBuffer};
 use crate::trusted_len::TrustedLen;
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 mod data;
 mod ffi;
 pub(super) mod fmt;

crates/nano-arrow/src/bitmap/immutable.rs

@@ -427,7 +427,7 @@ impl Bitmap {
     /// Create a new [`Bitmap`] from an arrow [`NullBuffer`]
     ///
     /// [`NullBuffer`]: arrow_buffer::buffer::NullBuffer
-    #[cfg(feature = "arrow-rs")]
+    #[cfg(feature = "arrow_rs")]
     pub fn from_null_buffer(value: arrow_buffer::buffer::NullBuffer) -> Self {
         let offset = value.offset();
         let length = value.len();
@@ -459,7 +459,7 @@ impl IntoIterator for Bitmap {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<Bitmap> for arrow_buffer::buffer::NullBuffer {
     fn from(value: Bitmap) -> Self {
         let null_count = value.unset_bits;

crates/nano-arrow/src/buffer/immutable.rs

@@ -310,14 +310,14 @@ impl<T: Copy> IntoIterator for Buffer<T> {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl<T: crate::types::NativeType> From<arrow_buffer::Buffer> for Buffer<T> {
     fn from(value: arrow_buffer::Buffer) -> Self {
         Self::from_bytes(crate::buffer::to_bytes(value))
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl<T: crate::types::NativeType> From<Buffer<T>> for arrow_buffer::Buffer {
     fn from(value: Buffer<T>) -> Self {
         crate::buffer::to_buffer(value.data).slice_with_length(

crates/nano-arrow/src/buffer/mod.rs

@@ -10,7 +10,7 @@ use crate::ffi::InternalArrowArray;
 pub(crate) enum BytesAllocator {
     InternalArrowArray(InternalArrowArray),
 
-    #[cfg(feature = "arrow-rs")]
+    #[cfg(feature = "arrow_rs")]
     Arrow(arrow_buffer::Buffer),
 }
 pub(crate) type BytesInner<T> = foreign_vec::ForeignVec<BytesAllocator, T>;
@@ -65,7 +65,7 @@ impl<T> From<BytesInner<T>> for Bytes<T> {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 pub(crate) fn to_buffer<T: crate::types::NativeType>(
     value: std::sync::Arc<Bytes<T>>,
 ) -> arrow_buffer::Buffer {
@@ -76,7 +76,7 @@ pub(crate) fn to_buffer<T: crate::types::NativeType>(
     unsafe { arrow_buffer::Buffer::from_custom_allocation(ptr, len, value) }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 pub(crate) fn to_bytes<T: crate::types::NativeType>(value: arrow_buffer::Buffer) -> Bytes<T> {
     let ptr = value.as_ptr();
     let align = ptr.align_offset(std::mem::align_of::<T>());

crates/nano-arrow/src/datatypes/field.rs

@@ -53,22 +53,22 @@ impl Field {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<Field> for arrow_schema::Field {
     fn from(value: Field) -> Self {
         Self::new(value.name, value.data_type.into(), value.is_nullable)
             .with_metadata(value.metadata.into_iter().collect())
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_schema::Field> for Field {
     fn from(value: arrow_schema::Field) -> Self {
         (&value).into()
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<&arrow_schema::Field> for Field {
     fn from(value: &arrow_schema::Field) -> Self {
         let data_type = value.data_type().clone().into();
@@ -81,14 +81,14 @@ impl From<&arrow_schema::Field> for Field {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_schema::FieldRef> for Field {
     fn from(value: arrow_schema::FieldRef) -> Self {
         value.as_ref().into()
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<&arrow_schema::FieldRef> for Field {
     fn from(value: &arrow_schema::FieldRef) -> Self {
         value.as_ref().into()

crates/nano-arrow/src/datatypes/mod.rs

@@ -159,7 +159,7 @@ pub enum DataType {
     Extension(String, Box<DataType>, Option<String>),
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<DataType> for arrow_schema::DataType {
     fn from(value: DataType) -> Self {
         use arrow_schema::{Field as ArrowField, UnionFields};
@@ -218,7 +218,7 @@ impl From<DataType> for arrow_schema::DataType {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_schema::DataType> for DataType {
     fn from(value: arrow_schema::DataType) -> Self {
         use arrow_schema::DataType;
@@ -291,7 +291,7 @@ pub enum UnionMode {
     Sparse,
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<UnionMode> for arrow_schema::UnionMode {
     fn from(value: UnionMode) -> Self {
         match value {
@@ -301,7 +301,7 @@ impl From<UnionMode> for arrow_schema::UnionMode {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_schema::UnionMode> for UnionMode {
     fn from(value: arrow_schema::UnionMode) -> Self {
         match value {
@@ -347,7 +347,7 @@ pub enum TimeUnit {
     Nanosecond,
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<TimeUnit> for arrow_schema::TimeUnit {
     fn from(value: TimeUnit) -> Self {
         match value {
@@ -359,7 +359,7 @@ impl From<TimeUnit> for arrow_schema::TimeUnit {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_schema::TimeUnit> for TimeUnit {
     fn from(value: arrow_schema::TimeUnit) -> Self {
         match value {
@@ -384,7 +384,7 @@ pub enum IntervalUnit {
     MonthDayNano,
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<IntervalUnit> for arrow_schema::IntervalUnit {
     fn from(value: IntervalUnit) -> Self {
         match value {
@@ -395,7 +395,7 @@ impl From<IntervalUnit> for arrow_schema::IntervalUnit {
     }
 }
 
-#[cfg(feature = "arrow-rs")]
+#[cfg(feature = "arrow_rs")]
 impl From<arrow_schema::IntervalUnit> for IntervalUnit {
     fn from(value: arrow_schema::IntervalUnit) -> Self {
         match value {

crates/polars-core/Cargo.toml

@@ -16,6 +16,7 @@ polars-utils = { workspace = true }
 
 ahash = { workspace = true }
 arrow = { workspace = true }
+arrow-array = { workspace = true, optional = true }
 bitflags = { workspace = true }
 bytemuck = { workspace = true }
 chrono = { workspace = true, optional = true }
@@ -108,6 +109,7 @@ chunked_ids = []
 describe = []
 timezones = ["chrono-tz", "arrow/chrono-tz", "polars-arrow/timezones"]
 dynamic_group_by = ["dtype-datetime", "dtype-date"]
+arrow_rs = ["arrow-array", "arrow/arrow_rs"]
 
 # opt-in datatypes for Series
 dtype-date = ["temporal"]