Skip to content

Commit 5909bfd

Browse files
v0y4g3rv0y4g3r
committed
feat: ignore timezone info when copy from external files
1 parent c850e96 commit 5909bfd

File tree

6 files changed

+200
-19
lines changed

6 files changed

+200
-19
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/datanode/src/error.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,11 +423,13 @@ pub enum Error {
423423
},
424424

425425
#[snafu(display(
426-
"File Schema mismatch, expected table schema: {} but found :{}",
426+
"File schema mismatch at index {}, expected table schema: {} but found :{}",
427+
index,
427428
table_schema,
428429
file_schema
429430
))]
430431
InvalidSchema {
432+
index: usize,
431433
table_schema: String,
432434
file_schema: String,
433435
},

‎src/datanode/src/sql/copy_table_from.rs

Lines changed: 138 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ use common_datasource::util::find_dir_and_filename;
2323
use common_query::Output;
2424
use common_recordbatch::error::DataTypesSnafu;
2525
use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
26+
use datatypes::arrow::datatypes::{DataType, SchemaRef};
27+
use datatypes::arrow::record_batch::RecordBatch;
2628
use datatypes::vectors::Helper;
2729
use futures_util::StreamExt;
2830
use regex::Regex;
29-
use snafu::{ensure, ResultExt};
31+
use snafu::ResultExt;
3032
use table::engine::TableReference;
3133
use table::requests::{CopyTableRequest, InsertRequest};
3234
use tokio::io::BufReader;
@@ -88,13 +90,7 @@ impl SqlHandler {
8890
.await
8991
.context(error::ReadParquetSnafu)?;
9092

91-
ensure!(
92-
builder.schema() == table.schema().arrow_schema(),
93-
error::InvalidSchemaSnafu {
94-
table_schema: table.schema().arrow_schema().to_string(),
95-
file_schema: (*(builder.schema())).to_string()
96-
}
97-
);
93+
ensure_schema_matches_ignore_timezone(builder.schema(), table.schema().arrow_schema())?;
9894

9995
let mut stream = builder
10096
.build()
@@ -159,3 +155,137 @@ async fn batch_insert(
159155
*pending_bytes = 0;
160156
Ok(res)
161157
}
158+
159+
fn ensure_schema_matches_ignore_timezone(left: &SchemaRef, right: &SchemaRef) -> Result<()> {
160+
let not_match = left
161+
.fields
162+
.iter()
163+
.zip(right.fields.iter())
164+
.map(|(l, r)| (l.data_type(), r.data_type()))
165+
.enumerate()
166+
.find(|(_, (l, r))| !data_type_equals_ignore_timezone(l, r));
167+
168+
if let Some((index, _)) = not_match {
169+
error::InvalidSchemaSnafu {
170+
index,
171+
table_schema: left.to_string(),
172+
file_schema: right.to_string(),
173+
}
174+
.fail()
175+
} else {
176+
Ok(())
177+
}
178+
}
179+
180+
fn data_type_equals_ignore_timezone(l: &DataType, r: &DataType) -> bool {
181+
match (l, r) {
182+
(DataType::List(a), DataType::List(b))
183+
| (DataType::LargeList(a), DataType::LargeList(b)) => {
184+
a.is_nullable() == b.is_nullable()
185+
&& data_type_equals_ignore_timezone(a.data_type(), b.data_type())
186+
}
187+
(DataType::FixedSizeList(a, a_size), DataType::FixedSizeList(b, b_size)) => {
188+
a_size == b_size
189+
&& a.is_nullable() == b.is_nullable()
190+
&& data_type_equals_ignore_timezone(a.data_type(), b.data_type())
191+
}
192+
(DataType::Struct(a), DataType::Struct(b)) => {
193+
a.len() == b.len()
194+
&& a.iter().zip(b).all(|(a, b)| {
195+
a.is_nullable() == b.is_nullable()
196+
&& data_type_equals_ignore_timezone(a.data_type(), b.data_type())
197+
})
198+
}
199+
(DataType::Map(a_field, a_is_sorted), DataType::Map(b_field, b_is_sorted)) => {
200+
a_field == b_field && a_is_sorted == b_is_sorted
201+
}
202+
(DataType::Timestamp(l_unit, _), DataType::Timestamp(r_unit, _)) => l_unit == r_unit,
203+
_ => l == r,
204+
}
205+
}
206+
207+
#[cfg(test)]
208+
mod tests {
209+
use std::sync::Arc;
210+
211+
use datatypes::arrow::datatypes::{Field, Schema};
212+
213+
use super::*;
214+
215+
fn test_schema_matches(l: (DataType, bool), r: (DataType, bool), matches: bool) {
216+
let s1 = Arc::new(Schema::new(vec![Field::new("col", l.0, l.1)]));
217+
let s2 = Arc::new(Schema::new(vec![Field::new("col", r.0, r.1)]));
218+
let res = ensure_schema_matches_ignore_timezone(&s1, &s2);
219+
assert_eq!(matches, res.is_ok())
220+
}
221+
222+
#[test]
223+
fn test_ensure_datatype_matches_ignore_timezone() {
224+
test_schema_matches(
225+
(
226+
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
227+
true,
228+
),
229+
(
230+
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
231+
true,
232+
),
233+
true,
234+
);
235+
236+
test_schema_matches(
237+
(
238+
DataType::Timestamp(
239+
datatypes::arrow::datatypes::TimeUnit::Second,
240+
Some("UTC".to_string()),
241+
),
242+
true,
243+
),
244+
(
245+
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
246+
true,
247+
),
248+
true,
249+
);
250+
251+
test_schema_matches(
252+
(
253+
DataType::Timestamp(
254+
datatypes::arrow::datatypes::TimeUnit::Second,
255+
Some("UTC".to_string()),
256+
),
257+
true,
258+
),
259+
(
260+
DataType::Timestamp(
261+
datatypes::arrow::datatypes::TimeUnit::Second,
262+
Some("PDT".to_string()),
263+
),
264+
true,
265+
),
266+
true,
267+
);
268+
269+
test_schema_matches(
270+
(
271+
DataType::Timestamp(
272+
datatypes::arrow::datatypes::TimeUnit::Second,
273+
Some("UTC".to_string()),
274+
),
275+
true,
276+
),
277+
(
278+
DataType::Timestamp(
279+
datatypes::arrow::datatypes::TimeUnit::Millisecond,
280+
Some("UTC".to_string()),
281+
),
282+
true,
283+
),
284+
false,
285+
);
286+
287+
test_schema_matches((DataType::Int8, true), (DataType::Int8, true), true);
288+
289+
test_schema_matches((DataType::Int8, true), (DataType::Int16, true), false);
290+
}
291+
}

‎src/datatypes/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ test = []
1010

1111
[dependencies]
1212
arrow.workspace = true
13+
arrow-array = "36"
1314
arrow-schema.workspace = true
1415
common-base = { path = "../common/base" }
1516
common-error = { path = "../common/error" }

‎src/datatypes/src/vectors/helper.rs

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -238,16 +238,18 @@ impl Helper {
238238
ArrowDataType::Date64 => Arc::new(DateTimeVector::try_from_arrow_array(array)?),
239239
ArrowDataType::List(_) => Arc::new(ListVector::try_from_arrow_array(array)?),
240240
ArrowDataType::Timestamp(unit, _) => match unit {
241-
TimeUnit::Second => Arc::new(TimestampSecondVector::try_from_arrow_array(array)?),
242-
TimeUnit::Millisecond => {
243-
Arc::new(TimestampMillisecondVector::try_from_arrow_array(array)?)
244-
}
245-
TimeUnit::Microsecond => {
246-
Arc::new(TimestampMicrosecondVector::try_from_arrow_array(array)?)
247-
}
248-
TimeUnit::Nanosecond => {
249-
Arc::new(TimestampNanosecondVector::try_from_arrow_array(array)?)
250-
}
241+
TimeUnit::Second => Arc::new(
242+
TimestampSecondVector::try_from_arrow_timestamp_array(array)?,
243+
),
244+
TimeUnit::Millisecond => Arc::new(
245+
TimestampMillisecondVector::try_from_arrow_timestamp_array(array)?,
246+
),
247+
TimeUnit::Microsecond => Arc::new(
248+
TimestampMicrosecondVector::try_from_arrow_timestamp_array(array)?,
249+
),
250+
TimeUnit::Nanosecond => Arc::new(
251+
TimestampNanosecondVector::try_from_arrow_timestamp_array(array)?,
252+
),
251253
},
252254
ArrowDataType::Float16
253255
| ArrowDataType::Time32(_)

‎src/datatypes/src/vectors/primitive.rs

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ use std::sync::Arc;
1818

1919
use arrow::array::{
2020
Array, ArrayBuilder, ArrayData, ArrayIter, ArrayRef, PrimitiveArray, PrimitiveBuilder,
21+
TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
22+
TimestampSecondArray,
2123
};
24+
use arrow_schema::DataType;
2225
use serde_json::Value as JsonValue;
2326
use snafu::OptionExt;
2427

@@ -70,6 +73,48 @@ impl<T: LogicalPrimitiveType> PrimitiveVector<T> {
7073
Ok(Self::new(concrete_array))
7174
}
7275

76+
/// Converts arrow timestamp array to vectors, ignoring time zone info.
77+
pub fn try_from_arrow_timestamp_array(array: impl AsRef<dyn Array>) -> Result<Self> {
78+
let array = array.as_ref();
79+
let array_data = match array.data_type() {
80+
DataType::Timestamp(unit, _) => match unit {
81+
arrow_schema::TimeUnit::Second => array
82+
.as_any()
83+
.downcast_ref::<TimestampSecondArray>()
84+
.unwrap()
85+
.with_timezone_opt(None)
86+
.data()
87+
.clone(),
88+
arrow_schema::TimeUnit::Millisecond => array
89+
.as_any()
90+
.downcast_ref::<TimestampMillisecondArray>()
91+
.unwrap()
92+
.with_timezone_opt(None)
93+
.data()
94+
.clone(),
95+
arrow_schema::TimeUnit::Microsecond => array
96+
.as_any()
97+
.downcast_ref::<TimestampMicrosecondArray>()
98+
.unwrap()
99+
.with_timezone_opt(None)
100+
.data()
101+
.clone(),
102+
arrow_schema::TimeUnit::Nanosecond => array
103+
.as_any()
104+
.downcast_ref::<TimestampNanosecondArray>()
105+
.unwrap()
106+
.with_timezone_opt(None)
107+
.data()
108+
.clone(),
109+
},
110+
_ => {
111+
unreachable!()
112+
}
113+
};
114+
let concrete_array = PrimitiveArray::<T::ArrowPrimitive>::from(array_data);
115+
Ok(Self::new(concrete_array))
116+
}
117+
73118
pub fn from_slice<P: AsRef<[T::Native]>>(slice: P) -> Self {
74119
let iter = slice.as_ref().iter().copied();
75120
Self {

0 commit comments

Comments
 (0)