Skip to content

Commit b7bdee6

Browse files
v0y4g3rv0y4g3r
authored
feat: ignoring time zone info when import from external files (#1341)
* feat: ignore timezone info when copy from external files * chore: rebase onto develop
1 parent c850e96 commit b7bdee6

File tree

6 files changed

+199
-19
lines changed

6 files changed

+199
-19
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/datanode/src/error.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,11 +423,13 @@ pub enum Error {
423423
},
424424

425425
#[snafu(display(
426-
"File Schema mismatch, expected table schema: {} but found :{}",
426+
"File schema mismatch at index {}, expected table schema: {} but found: {}",
427+
index,
427428
table_schema,
428429
file_schema
429430
))]
430431
InvalidSchema {
432+
index: usize,
431433
table_schema: String,
432434
file_schema: String,
433435
},

‎src/datanode/src/sql/copy_table_from.rs

Lines changed: 137 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,11 @@ use common_datasource::util::find_dir_and_filename;
2323
use common_query::Output;
2424
use common_recordbatch::error::DataTypesSnafu;
2525
use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
26+
use datatypes::arrow::datatypes::{DataType, SchemaRef};
2627
use datatypes::vectors::Helper;
2728
use futures_util::StreamExt;
2829
use regex::Regex;
29-
use snafu::{ensure, ResultExt};
30+
use snafu::ResultExt;
3031
use table::engine::TableReference;
3132
use table::requests::{CopyTableRequest, InsertRequest};
3233
use tokio::io::BufReader;
@@ -88,13 +89,7 @@ impl SqlHandler {
8889
.await
8990
.context(error::ReadParquetSnafu)?;
9091

91-
ensure!(
92-
builder.schema() == table.schema().arrow_schema(),
93-
error::InvalidSchemaSnafu {
94-
table_schema: table.schema().arrow_schema().to_string(),
95-
file_schema: (*(builder.schema())).to_string()
96-
}
97-
);
92+
ensure_schema_matches_ignore_timezone(builder.schema(), table.schema().arrow_schema())?;
9893

9994
let mut stream = builder
10095
.build()
@@ -159,3 +154,137 @@ async fn batch_insert(
159154
*pending_bytes = 0;
160155
Ok(res)
161156
}
157+
158+
fn ensure_schema_matches_ignore_timezone(left: &SchemaRef, right: &SchemaRef) -> Result<()> {
159+
let not_match = left
160+
.fields
161+
.iter()
162+
.zip(right.fields.iter())
163+
.map(|(l, r)| (l.data_type(), r.data_type()))
164+
.enumerate()
165+
.find(|(_, (l, r))| !data_type_equals_ignore_timezone(l, r));
166+
167+
if let Some((index, _)) = not_match {
168+
error::InvalidSchemaSnafu {
169+
index,
170+
table_schema: left.to_string(),
171+
file_schema: right.to_string(),
172+
}
173+
.fail()
174+
} else {
175+
Ok(())
176+
}
177+
}
178+
179+
fn data_type_equals_ignore_timezone(l: &DataType, r: &DataType) -> bool {
180+
match (l, r) {
181+
(DataType::List(a), DataType::List(b))
182+
| (DataType::LargeList(a), DataType::LargeList(b)) => {
183+
a.is_nullable() == b.is_nullable()
184+
&& data_type_equals_ignore_timezone(a.data_type(), b.data_type())
185+
}
186+
(DataType::FixedSizeList(a, a_size), DataType::FixedSizeList(b, b_size)) => {
187+
a_size == b_size
188+
&& a.is_nullable() == b.is_nullable()
189+
&& data_type_equals_ignore_timezone(a.data_type(), b.data_type())
190+
}
191+
(DataType::Struct(a), DataType::Struct(b)) => {
192+
a.len() == b.len()
193+
&& a.iter().zip(b).all(|(a, b)| {
194+
a.is_nullable() == b.is_nullable()
195+
&& data_type_equals_ignore_timezone(a.data_type(), b.data_type())
196+
})
197+
}
198+
(DataType::Map(a_field, a_is_sorted), DataType::Map(b_field, b_is_sorted)) => {
199+
a_field == b_field && a_is_sorted == b_is_sorted
200+
}
201+
(DataType::Timestamp(l_unit, _), DataType::Timestamp(r_unit, _)) => l_unit == r_unit,
202+
_ => l == r,
203+
}
204+
}
205+
206+
#[cfg(test)]
207+
mod tests {
208+
use std::sync::Arc;
209+
210+
use datatypes::arrow::datatypes::{Field, Schema};
211+
212+
use super::*;
213+
214+
fn test_schema_matches(l: (DataType, bool), r: (DataType, bool), matches: bool) {
215+
let s1 = Arc::new(Schema::new(vec![Field::new("col", l.0, l.1)]));
216+
let s2 = Arc::new(Schema::new(vec![Field::new("col", r.0, r.1)]));
217+
let res = ensure_schema_matches_ignore_timezone(&s1, &s2);
218+
assert_eq!(matches, res.is_ok())
219+
}
220+
221+
#[test]
222+
fn test_ensure_datatype_matches_ignore_timezone() {
223+
test_schema_matches(
224+
(
225+
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
226+
true,
227+
),
228+
(
229+
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
230+
true,
231+
),
232+
true,
233+
);
234+
235+
test_schema_matches(
236+
(
237+
DataType::Timestamp(
238+
datatypes::arrow::datatypes::TimeUnit::Second,
239+
Some("UTC".to_string()),
240+
),
241+
true,
242+
),
243+
(
244+
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
245+
true,
246+
),
247+
true,
248+
);
249+
250+
test_schema_matches(
251+
(
252+
DataType::Timestamp(
253+
datatypes::arrow::datatypes::TimeUnit::Second,
254+
Some("UTC".to_string()),
255+
),
256+
true,
257+
),
258+
(
259+
DataType::Timestamp(
260+
datatypes::arrow::datatypes::TimeUnit::Second,
261+
Some("PDT".to_string()),
262+
),
263+
true,
264+
),
265+
true,
266+
);
267+
268+
test_schema_matches(
269+
(
270+
DataType::Timestamp(
271+
datatypes::arrow::datatypes::TimeUnit::Second,
272+
Some("UTC".to_string()),
273+
),
274+
true,
275+
),
276+
(
277+
DataType::Timestamp(
278+
datatypes::arrow::datatypes::TimeUnit::Millisecond,
279+
Some("UTC".to_string()),
280+
),
281+
true,
282+
),
283+
false,
284+
);
285+
286+
test_schema_matches((DataType::Int8, true), (DataType::Int8, true), true);
287+
288+
test_schema_matches((DataType::Int8, true), (DataType::Int16, true), false);
289+
}
290+
}

‎src/datatypes/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ test = []
1010

1111
[dependencies]
1212
arrow.workspace = true
13+
arrow-array = "36"
1314
arrow-schema.workspace = true
1415
common-base = { path = "../common/base" }
1516
common-error = { path = "../common/error" }

‎src/datatypes/src/vectors/helper.rs

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -238,16 +238,18 @@ impl Helper {
238238
ArrowDataType::Date64 => Arc::new(DateTimeVector::try_from_arrow_array(array)?),
239239
ArrowDataType::List(_) => Arc::new(ListVector::try_from_arrow_array(array)?),
240240
ArrowDataType::Timestamp(unit, _) => match unit {
241-
TimeUnit::Second => Arc::new(TimestampSecondVector::try_from_arrow_array(array)?),
242-
TimeUnit::Millisecond => {
243-
Arc::new(TimestampMillisecondVector::try_from_arrow_array(array)?)
244-
}
245-
TimeUnit::Microsecond => {
246-
Arc::new(TimestampMicrosecondVector::try_from_arrow_array(array)?)
247-
}
248-
TimeUnit::Nanosecond => {
249-
Arc::new(TimestampNanosecondVector::try_from_arrow_array(array)?)
250-
}
241+
TimeUnit::Second => Arc::new(
242+
TimestampSecondVector::try_from_arrow_timestamp_array(array)?,
243+
),
244+
TimeUnit::Millisecond => Arc::new(
245+
TimestampMillisecondVector::try_from_arrow_timestamp_array(array)?,
246+
),
247+
TimeUnit::Microsecond => Arc::new(
248+
TimestampMicrosecondVector::try_from_arrow_timestamp_array(array)?,
249+
),
250+
TimeUnit::Nanosecond => Arc::new(
251+
TimestampNanosecondVector::try_from_arrow_timestamp_array(array)?,
252+
),
251253
},
252254
ArrowDataType::Float16
253255
| ArrowDataType::Time32(_)

‎src/datatypes/src/vectors/primitive.rs

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ use std::sync::Arc;
1818

1919
use arrow::array::{
2020
Array, ArrayBuilder, ArrayData, ArrayIter, ArrayRef, PrimitiveArray, PrimitiveBuilder,
21+
TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
22+
TimestampSecondArray,
2123
};
24+
use arrow_schema::DataType;
2225
use serde_json::Value as JsonValue;
2326
use snafu::OptionExt;
2427

@@ -70,6 +73,48 @@ impl<T: LogicalPrimitiveType> PrimitiveVector<T> {
7073
Ok(Self::new(concrete_array))
7174
}
7275

76+
/// Converts arrow timestamp array to vectors, ignoring time zone info.
77+
pub fn try_from_arrow_timestamp_array(array: impl AsRef<dyn Array>) -> Result<Self> {
78+
let array = array.as_ref();
79+
let array_data = match array.data_type() {
80+
DataType::Timestamp(unit, _) => match unit {
81+
arrow_schema::TimeUnit::Second => array
82+
.as_any()
83+
.downcast_ref::<TimestampSecondArray>()
84+
.unwrap()
85+
.with_timezone_opt(None)
86+
.data()
87+
.clone(),
88+
arrow_schema::TimeUnit::Millisecond => array
89+
.as_any()
90+
.downcast_ref::<TimestampMillisecondArray>()
91+
.unwrap()
92+
.with_timezone_opt(None)
93+
.data()
94+
.clone(),
95+
arrow_schema::TimeUnit::Microsecond => array
96+
.as_any()
97+
.downcast_ref::<TimestampMicrosecondArray>()
98+
.unwrap()
99+
.with_timezone_opt(None)
100+
.data()
101+
.clone(),
102+
arrow_schema::TimeUnit::Nanosecond => array
103+
.as_any()
104+
.downcast_ref::<TimestampNanosecondArray>()
105+
.unwrap()
106+
.with_timezone_opt(None)
107+
.data()
108+
.clone(),
109+
},
110+
_ => {
111+
unreachable!()
112+
}
113+
};
114+
let concrete_array = PrimitiveArray::<T::ArrowPrimitive>::from(array_data);
115+
Ok(Self::new(concrete_array))
116+
}
117+
73118
pub fn from_slice<P: AsRef<[T::Native]>>(slice: P) -> Self {
74119
let iter = slice.as_ref().iter().copied();
75120
Self {

0 commit comments

Comments
 (0)