Skip to content

Move make_date, to_char to datafusion-functions #9601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alamb merged 6 commits into from
Mar 14, 2024
Merged

Move make_date, to_char to datafusion-functions #9601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c5b0e7c
Move make_date, to_char to datafusion-functions
Omega359 Mar 11, 2024
e6c54df
Merge remote-tracking branch 'upstream/main' into feature/9538
Omega359 Mar 13, 2024
ebe3b2a
Update post merge to remove datetime_expressions.rs
Omega359 Mar 13, 2024
c9b67bd
Fix benchmarks.
Omega359 Mar 14, 2024
a25bc6d
Merge remote-tracking branch 'upstream/main' into feature/9538
Omega359 Mar 14, 2024
8c87ec6
Cargo fmt.
Omega359 Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions datafusion-cli/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

56 changes: 1 addition & 55 deletions datafusion/expr/src/built_in_function.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,10 @@ use std::fmt;
use std::str::FromStr;
use std::sync::{Arc, OnceLock};

use crate::signature::TIMEZONE_WILDCARD;
use crate::type_coercion::functions::data_types;
use crate::{FuncMonotonicity, Signature, TypeSignature, Volatility};

use arrow::datatypes::{DataType, Field, TimeUnit};
use arrow::datatypes::{DataType, Field};
use datafusion_common::{plan_err, DataFusionError, Result};

use strum::IntoEnumIterator;
Expand Down Expand Up @@ -190,8 +189,6 @@ pub enum BuiltinScalarFunction {
Substr,
/// to_hex
ToHex,
/// make_date
MakeDate,
/// translate
Translate,
/// trim
Expand All @@ -208,8 +205,6 @@ pub enum BuiltinScalarFunction {
SubstrIndex,
/// find_in_set
FindInSet,
/// to_char
ToChar,
}

/// Maps the sql function name to `BuiltinScalarFunction`
Expand Down Expand Up @@ -335,8 +330,6 @@ impl BuiltinScalarFunction {
BuiltinScalarFunction::Strpos => Volatility::Immutable,
BuiltinScalarFunction::Substr => Volatility::Immutable,
BuiltinScalarFunction::ToHex => Volatility::Immutable,
BuiltinScalarFunction::ToChar => Volatility::Immutable,
BuiltinScalarFunction::MakeDate => Volatility::Immutable,
BuiltinScalarFunction::Translate => Volatility::Immutable,
BuiltinScalarFunction::Trim => Volatility::Immutable,
BuiltinScalarFunction::Upper => Volatility::Immutable,
Expand Down Expand Up @@ -490,8 +483,6 @@ impl BuiltinScalarFunction {
BuiltinScalarFunction::FindInSet => {
utf8_to_int_type(&input_expr_types[0], "find_in_set")
}
BuiltinScalarFunction::ToChar => Ok(Utf8),
BuiltinScalarFunction::MakeDate => Ok(Date32),
BuiltinScalarFunction::Translate => {
utf8_to_str_type(&input_expr_types[0], "translate")
}
Expand Down Expand Up @@ -567,7 +558,6 @@ impl BuiltinScalarFunction {
/// Return the argument [`Signature`] supported by this function
pub fn signature(&self) -> Signature {
use DataType::*;
use TimeUnit::*;
use TypeSignature::*;
// note: the physical expression must accept the type returned by this function or the execution panics.

Expand Down Expand Up @@ -651,41 +641,6 @@ impl BuiltinScalarFunction {
vec![Exact(vec![Utf8, Int64]), Exact(vec![LargeUtf8, Int64])],
self.volatility(),
),
BuiltinScalarFunction::ToChar => Signature::one_of(
vec![
Exact(vec![Date32, Utf8]),
Exact(vec![Date64, Utf8]),
Exact(vec![Time32(Millisecond), Utf8]),
Exact(vec![Time32(Second), Utf8]),
Exact(vec![Time64(Microsecond), Utf8]),
Exact(vec![Time64(Nanosecond), Utf8]),
Exact(vec![Timestamp(Second, None), Utf8]),
Exact(vec![
Timestamp(Second, Some(TIMEZONE_WILDCARD.into())),
Utf8,
]),
Exact(vec![Timestamp(Millisecond, None), Utf8]),
Exact(vec![
Timestamp(Millisecond, Some(TIMEZONE_WILDCARD.into())),
Utf8,
]),
Exact(vec![Timestamp(Microsecond, None), Utf8]),
Exact(vec![
Timestamp(Microsecond, Some(TIMEZONE_WILDCARD.into())),
Utf8,
]),
Exact(vec![Timestamp(Nanosecond, None), Utf8]),
Exact(vec![
Timestamp(Nanosecond, Some(TIMEZONE_WILDCARD.into())),
Utf8,
]),
Exact(vec![Duration(Second), Utf8]),
Exact(vec![Duration(Millisecond), Utf8]),
Exact(vec![Duration(Microsecond), Utf8]),
Exact(vec![Duration(Nanosecond), Utf8]),
],
self.volatility(),
),
BuiltinScalarFunction::SplitPart => Signature::one_of(
vec![
Exact(vec![Utf8, Utf8, Int64]),
Expand Down Expand Up @@ -821,11 +776,6 @@ impl BuiltinScalarFunction {
// will be as good as the number of digits in the number
Signature::uniform(1, vec![Float64, Float32], self.volatility())
}
BuiltinScalarFunction::MakeDate => Signature::uniform(
3,
vec![Int32, Int64, UInt32, UInt64, Utf8],
self.volatility(),
),
BuiltinScalarFunction::Iszero => Signature::one_of(
vec![Exact(vec![Float32]), Exact(vec![Float64])],
self.volatility(),
Expand Down Expand Up @@ -943,10 +893,6 @@ impl BuiltinScalarFunction {
BuiltinScalarFunction::SubstrIndex => &["substr_index", "substring_index"],
BuiltinScalarFunction::FindInSet => &["find_in_set"],

// time/date functions
BuiltinScalarFunction::MakeDate => &["make_date"],
BuiltinScalarFunction::ToChar => &["to_char", "date_format"],

// hashing functions
BuiltinScalarFunction::ArrayElement => &[
"array_element",
Expand Down
8 changes: 0 additions & 8 deletions datafusion/expr/src/expr_fn.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -768,14 +768,6 @@ nary_scalar_expr!(
"replace the substring of string that starts at the start'th character and extends for count characters with new substring"
);

// date functions
scalar_expr!(
ToChar,
to_char,
datetime format,
"converts a date, time, timestamp or duration to a string based on the provided format"
);
scalar_expr!(MakeDate, make_date, year month day, "make a date from year, month and day component parts");
scalar_expr!(Nanvl, nanvl, x y, "returns x if x is not NaN otherwise returns y");
scalar_expr!(
Iszero,
Expand Down
9 changes: 9 additions & 0 deletions datafusion/functions/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ path = "src/lib.rs"
[dependencies]
arrow = { workspace = true }
arrow-array = { workspace = true }
arrow-schema = { workspace = true }
base64 = { version = "0.22", optional = true }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
Expand Down Expand Up @@ -85,3 +86,11 @@ name = "to_timestamp"
[[bench]]
harness = false
name = "regx"

[[bench]]
harness = false
name = "make_date"

[[bench]]
harness = false
name = "to_char"
14 changes: 9 additions & 5 deletions ...fusion/physical-expr/benches/make_date.rs → datafusion/functions/benches/make_date.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use rand::Rng;

use datafusion_common::ScalarValue;
use datafusion_expr::ColumnarValue;
use datafusion_physical_expr::datetime_expressions::make_date;
use datafusion_functions::datetime::make_date;

fn years(rng: &mut ThreadRng) -> Int32Array {
let mut years = vec![];
Expand Down Expand Up @@ -63,7 +63,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
make_date(&[years.clone(), months.clone(), days.clone()])
make_date()
.invoke(&[years.clone(), months.clone(), days.clone()])
.expect("make_date should work on valid values"),
)
})
Expand All @@ -77,7 +78,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
make_date(&[year.clone(), months.clone(), days.clone()])
make_date()
.invoke(&[year.clone(), months.clone(), days.clone()])
.expect("make_date should work on valid values"),
)
})
Expand All @@ -91,7 +93,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
make_date(&[year.clone(), month.clone(), days.clone()])
make_date()
.invoke(&[year.clone(), month.clone(), days.clone()])
.expect("make_date should work on valid values"),
)
})
Expand All @@ -104,7 +107,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
make_date(&[year.clone(), month.clone(), day.clone()])
make_date()
.invoke(&[year.clone(), month.clone(), day.clone()])
.expect("make_date should work on valid values"),
)
})
Expand Down
11 changes: 7 additions & 4 deletions datafusion/physical-expr/benches/to_char.rs → datafusion/functions/benches/to_char.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use rand::Rng;
use datafusion_common::ScalarValue;
use datafusion_common::ScalarValue::TimestampNanosecond;
use datafusion_expr::ColumnarValue;
use datafusion_physical_expr::datetime_expressions::to_char;
use datafusion_functions::datetime::to_char;

fn random_date_in_range(
rng: &mut ThreadRng,
Expand Down Expand Up @@ -87,7 +87,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
to_char(&[data.clone(), patterns.clone()])
to_char()
.invoke(&[data.clone(), patterns.clone()])
.expect("to_char should work on valid values"),
)
})
Expand All @@ -101,7 +102,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
to_char(&[data.clone(), patterns.clone()])
to_char()
.invoke(&[data.clone(), patterns.clone()])
.expect("to_char should work on valid values"),
)
})
Expand All @@ -123,7 +125,8 @@ fn criterion_benchmark(c: &mut Criterion) {

b.iter(|| {
black_box(
to_char(&[data.clone(), pattern.clone()])
to_char()
.invoke(&[data.clone(), pattern.clone()])
.expect("to_char should work on valid values"),
)
})
Expand Down
Loading