Skip to content

Commit acddecb

Browse files
Omega359mustafasrepoalamb
authored
Move date_part, date_trunc, date_bin functions to datafusion-functions (#9435)
* Move date_part, date_trunc, date_bin functions to datafusion-functions * I do not understand why the logical plan changed but updating the explain text to reflect the change. The physical plan is unchanged. * Fix fmt * Improvements to remove datafusion-functions dependency from sq and physical-expr * Fix function arguments for date_bin, date_trunc and date_part. * Fix projection change. Add new test date_bin monotonicity --------- Co-authored-by: Mustafa Akur <[email protected]> Co-authored-by: Andrew Lamb <[email protected]>
1 parent 6710e6d commit acddecb

File tree

19 files changed

+1981
-1740
lines changed

19 files changed

+1981
-1740
lines changed

‎datafusion/expr/src/built_in_function.rs

Lines changed: 1 addition & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use crate::type_coercion::binary::get_wider_type;
2828
use crate::type_coercion::functions::data_types;
2929
use crate::{FuncMonotonicity, Signature, TypeSignature, Volatility};
3030

31-
use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit};
31+
use arrow::datatypes::{DataType, Field, Fields, TimeUnit};
3232
use datafusion_common::{plan_err, DataFusionError, Result};
3333

3434
use strum::IntoEnumIterator;
@@ -180,12 +180,6 @@ pub enum BuiltinScalarFunction {
180180
Concat,
181181
/// concat_ws
182182
ConcatWithSeparator,
183-
/// date_part
184-
DatePart,
185-
/// date_trunc
186-
DateTrunc,
187-
/// date_bin
188-
DateBin,
189183
/// ends_with
190184
EndsWith,
191185
/// initcap
@@ -382,9 +376,6 @@ impl BuiltinScalarFunction {
382376
BuiltinScalarFunction::Chr => Volatility::Immutable,
383377
BuiltinScalarFunction::Concat => Volatility::Immutable,
384378
BuiltinScalarFunction::ConcatWithSeparator => Volatility::Immutable,
385-
BuiltinScalarFunction::DatePart => Volatility::Immutable,
386-
BuiltinScalarFunction::DateTrunc => Volatility::Immutable,
387-
BuiltinScalarFunction::DateBin => Volatility::Immutable,
388379
BuiltinScalarFunction::EndsWith => Volatility::Immutable,
389380
BuiltinScalarFunction::InitCap => Volatility::Immutable,
390381
BuiltinScalarFunction::Left => Volatility::Immutable,
@@ -592,27 +583,6 @@ impl BuiltinScalarFunction {
592583
}
593584
BuiltinScalarFunction::Concat => Ok(Utf8),
594585
BuiltinScalarFunction::ConcatWithSeparator => Ok(Utf8),
595-
BuiltinScalarFunction::DatePart => Ok(Float64),
596-
BuiltinScalarFunction::DateBin | BuiltinScalarFunction::DateTrunc => {
597-
match &input_expr_types[1] {
598-
Timestamp(Nanosecond, None) | Utf8 | Null => {
599-
Ok(Timestamp(Nanosecond, None))
600-
}
601-
Timestamp(Nanosecond, tz_opt) => {
602-
Ok(Timestamp(Nanosecond, tz_opt.clone()))
603-
}
604-
Timestamp(Microsecond, tz_opt) => {
605-
Ok(Timestamp(Microsecond, tz_opt.clone()))
606-
}
607-
Timestamp(Millisecond, tz_opt) => {
608-
Ok(Timestamp(Millisecond, tz_opt.clone()))
609-
}
610-
Timestamp(Second, tz_opt) => Ok(Timestamp(Second, tz_opt.clone())),
611-
_ => plan_err!(
612-
"The {self} function can only accept timestamp as the second arg."
613-
),
614-
}
615-
}
616586
BuiltinScalarFunction::InitCap => {
617587
utf8_to_str_type(&input_expr_types[0], "initcap")
618588
}
@@ -784,7 +754,6 @@ impl BuiltinScalarFunction {
784754
/// Return the argument [`Signature`] supported by this function
785755
pub fn signature(&self) -> Signature {
786756
use DataType::*;
787-
use IntervalUnit::*;
788757
use TimeUnit::*;
789758
use TypeSignature::*;
790759
// note: the physical expression must accept the type returned by this function or the execution panics.
@@ -947,108 +916,6 @@ impl BuiltinScalarFunction {
947916
],
948917
self.volatility(),
949918
),
950-
BuiltinScalarFunction::DateTrunc => Signature::one_of(
951-
vec![
952-
Exact(vec![Utf8, Timestamp(Nanosecond, None)]),
953-
Exact(vec![
954-
Utf8,
955-
Timestamp(Nanosecond, Some(TIMEZONE_WILDCARD.into())),
956-
]),
957-
Exact(vec![Utf8, Timestamp(Microsecond, None)]),
958-
Exact(vec![
959-
Utf8,
960-
Timestamp(Microsecond, Some(TIMEZONE_WILDCARD.into())),
961-
]),
962-
Exact(vec![Utf8, Timestamp(Millisecond, None)]),
963-
Exact(vec![
964-
Utf8,
965-
Timestamp(Millisecond, Some(TIMEZONE_WILDCARD.into())),
966-
]),
967-
Exact(vec![Utf8, Timestamp(Second, None)]),
968-
Exact(vec![
969-
Utf8,
970-
Timestamp(Second, Some(TIMEZONE_WILDCARD.into())),
971-
]),
972-
],
973-
self.volatility(),
974-
),
975-
BuiltinScalarFunction::DateBin => {
976-
let base_sig = |array_type: TimeUnit| {
977-
vec![
978-
Exact(vec![
979-
Interval(MonthDayNano),
980-
Timestamp(array_type.clone(), None),
981-
Timestamp(Nanosecond, None),
982-
]),
983-
Exact(vec![
984-
Interval(MonthDayNano),
985-
Timestamp(array_type.clone(), Some(TIMEZONE_WILDCARD.into())),
986-
Timestamp(Nanosecond, Some(TIMEZONE_WILDCARD.into())),
987-
]),
988-
Exact(vec![
989-
Interval(DayTime),
990-
Timestamp(array_type.clone(), None),
991-
Timestamp(Nanosecond, None),
992-
]),
993-
Exact(vec![
994-
Interval(DayTime),
995-
Timestamp(array_type.clone(), Some(TIMEZONE_WILDCARD.into())),
996-
Timestamp(Nanosecond, Some(TIMEZONE_WILDCARD.into())),
997-
]),
998-
Exact(vec![
999-
Interval(MonthDayNano),
1000-
Timestamp(array_type.clone(), None),
1001-
]),
1002-
Exact(vec![
1003-
Interval(MonthDayNano),
1004-
Timestamp(array_type.clone(), Some(TIMEZONE_WILDCARD.into())),
1005-
]),
1006-
Exact(vec![
1007-
Interval(DayTime),
1008-
Timestamp(array_type.clone(), None),
1009-
]),
1010-
Exact(vec![
1011-
Interval(DayTime),
1012-
Timestamp(array_type, Some(TIMEZONE_WILDCARD.into())),
1013-
]),
1014-
]
1015-
};
1016-
1017-
let full_sig = [Nanosecond, Microsecond, Millisecond, Second]
1018-
.into_iter()
1019-
.map(base_sig)
1020-
.collect::<Vec<_>>()
1021-
.concat();
1022-
1023-
Signature::one_of(full_sig, self.volatility())
1024-
}
1025-
BuiltinScalarFunction::DatePart => Signature::one_of(
1026-
vec![
1027-
Exact(vec![Utf8, Timestamp(Nanosecond, None)]),
1028-
Exact(vec![
1029-
Utf8,
1030-
Timestamp(Nanosecond, Some(TIMEZONE_WILDCARD.into())),
1031-
]),
1032-
Exact(vec![Utf8, Timestamp(Millisecond, None)]),
1033-
Exact(vec![
1034-
Utf8,
1035-
Timestamp(Millisecond, Some(TIMEZONE_WILDCARD.into())),
1036-
]),
1037-
Exact(vec![Utf8, Timestamp(Microsecond, None)]),
1038-
Exact(vec![
1039-
Utf8,
1040-
Timestamp(Microsecond, Some(TIMEZONE_WILDCARD.into())),
1041-
]),
1042-
Exact(vec![Utf8, Timestamp(Second, None)]),
1043-
Exact(vec![
1044-
Utf8,
1045-
Timestamp(Second, Some(TIMEZONE_WILDCARD.into())),
1046-
]),
1047-
Exact(vec![Utf8, Date64]),
1048-
Exact(vec![Utf8, Date32]),
1049-
],
1050-
self.volatility(),
1051-
),
1052919
BuiltinScalarFunction::SplitPart => Signature::one_of(
1053920
vec![
1054921
Exact(vec![Utf8, Utf8, Int64]),
@@ -1240,11 +1107,6 @@ impl BuiltinScalarFunction {
12401107
| BuiltinScalarFunction::Pi
12411108
) {
12421109
Some(vec![Some(true)])
1243-
} else if matches!(
1244-
&self,
1245-
BuiltinScalarFunction::DateTrunc | BuiltinScalarFunction::DateBin
1246-
) {
1247-
Some(vec![None, Some(true)])
12481110
} else if *self == BuiltinScalarFunction::Log {
12491111
Some(vec![Some(true), Some(false)])
12501112
} else {
@@ -1337,9 +1199,6 @@ impl BuiltinScalarFunction {
13371199
BuiltinScalarFunction::CurrentDate => &["current_date", "today"],
13381200
BuiltinScalarFunction::CurrentTime => &["current_time"],
13391201
BuiltinScalarFunction::MakeDate => &["make_date"],
1340-
BuiltinScalarFunction::DateBin => &["date_bin"],
1341-
BuiltinScalarFunction::DateTrunc => &["date_trunc", "datetrunc"],
1342-
BuiltinScalarFunction::DatePart => &["date_part", "datepart"],
13431202
BuiltinScalarFunction::ToChar => &["to_char", "date_format"],
13441203
BuiltinScalarFunction::FromUnixtime => &["from_unixtime"],
13451204

‎datafusion/expr/src/expr.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,11 @@ impl ScalarFunction {
419419
args,
420420
}
421421
}
422+
423+
/// Create a new ScalarFunction expression with a user-defined function (UDF)
424+
pub fn new_func_def(func_def: ScalarFunctionDefinition, args: Vec<Expr>) -> Self {
425+
Self { func_def, args }
426+
}
422427
}
423428

424429
/// Access a sub field of a nested type, such as `Field` or `List`

‎datafusion/expr/src/expr_fn.rs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -819,9 +819,6 @@ nary_scalar_expr!(
819819
);
820820

821821
// date functions
822-
scalar_expr!(DatePart, date_part, part date, "extracts a subfield from the date");
823-
scalar_expr!(DateTrunc, date_trunc, part date, "truncates the date to a specified level of precision");
824-
scalar_expr!(DateBin, date_bin, stride source origin, "coerces an arbitrary timestamp to the start of the nearest specified interval");
825822
scalar_expr!(
826823
ToChar,
827824
to_char,
@@ -1309,9 +1306,6 @@ mod test {
13091306
test_scalar_expr!(Trim, trim, string);
13101307
test_scalar_expr!(Upper, upper, string);
13111308

1312-
test_scalar_expr!(DatePart, date_part, part, date);
1313-
test_scalar_expr!(DateTrunc, date_trunc, part, date);
1314-
test_scalar_expr!(DateBin, date_bin, stride, source, origin);
13151309
test_scalar_expr!(FromUnixtime, from_unixtime, unixtime);
13161310

13171311
test_scalar_expr!(ArrayAppend, array_append, array, element);

0 commit comments

Comments
 (0)