From 80d06c317c933d6b2a135f1849ba5996eb5027d1 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 2 Dec 2020 16:07:31 +0800 Subject: [PATCH 1/9] add new functions --- .../catalyst/analysis/FunctionRegistry.scala | 3 + .../expressions/datetimeExpressions.scala | 73 +++++++++++++++++++ .../expressions/DateExpressionsSuite.scala | 32 ++++++++ .../sql-functions/sql-expression-schema.md | 5 +- .../resources/sql-tests/inputs/datetime.sql | 4 + .../sql-tests/results/ansi/datetime.sql.out | 26 ++++++- .../sql-tests/results/datetime.sql.out | 26 ++++++- 7 files changed, 166 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 6fb9bed9625d5..5c2816a0baa95 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -430,6 +430,9 @@ object FunctionRegistry { expression[SecondsToTimestamp]("timestamp_seconds"), expression[MillisToTimestamp]("timestamp_millis"), expression[MicrosToTimestamp]("timestamp_micros"), + expression[UnixSeconds]("unix_seconds"), + expression[UnixMillis]("unix_millis"), + expression[UnixMicros]("unix_micros"), // collection functions expression[CreateArray]("array"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 1ff5833fb4dd6..294e1e4e035ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -524,6 +524,79 @@ case class MicrosToTimestamp(child: Expression) override def prettyName: String = "timestamp_micros" } +abstract class TimestampToLongBase extends UnaryExpression + with ExpectsInputTypes with NullIntolerant { + + protected def scaleFactor: Long + + override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType) + + override def dataType: DataType = LongType + + override def nullSafeEval(input: Any): Any = { + Math.floorDiv(input.asInstanceOf[Number].longValue(), scaleFactor) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + if (scaleFactor == 1) { + defineCodeGen(ctx, ev, c => c) + } else { + defineCodeGen(ctx, ev, c => s"java.lang.Math.floorDiv($c, ${scaleFactor}L)") + } + } +} + +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + examples = """ + Examples: + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); + 1 + """, + group = "datetime_funcs", + since = "3.1.0") +// scalastyle:on line.size.limit +case class UnixSeconds(child: Expression) extends TimestampToLongBase { + override def scaleFactor: Long = MICROS_PER_SECOND + + override def prettyName: String = "unix_seconds" +} + +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + examples = """ + Examples: + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); + 1000 + """, + group = "datetime_funcs", + since = "3.1.0") +// scalastyle:on line.size.limit +case class UnixMillis(child: Expression) extends TimestampToLongBase { + override def scaleFactor: Long = MICROS_PER_MILLIS + + override def prettyName: String = "unix_millis" +} + +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + examples = """ + Examples: + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); + 1000000 + """, + group = "datetime_funcs", + since = "3.1.0") +// scalastyle:on line.size.limit +case class UnixMicros(child: Expression) extends TimestampToLongBase { + override def scaleFactor: Long = 1L + + override def prettyName: String = "unix_micros" +} + @ExpressionDescription( usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", examples = """ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index a3ffc1129fd5e..ca5f4ed987d6b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1197,6 +1197,38 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkResult(Int.MinValue.toLong - 100) } + test("UNIX_SECONDS") { + var timestamp = Literal(new Timestamp(0L)) + checkEvaluation(UnixSeconds(timestamp), 0L) + timestamp = Literal(new Timestamp(1000L)) + checkEvaluation(UnixSeconds(timestamp), 1L) + // Truncates higher levels of precision + timestamp = Literal(new Timestamp(1999L)) + checkEvaluation(UnixSeconds(timestamp), 1L) + } + + test("UNIX_MILLS") { + var timestamp = Literal(new Timestamp(0L)) + checkEvaluation(UnixMillis(timestamp), 0L) + timestamp = Literal(new Timestamp(1000L)) + checkEvaluation(UnixMillis(timestamp), 1000L) + // Truncates higher levels of precision + val timestampWithNanos = new Timestamp(1000L) + timestampWithNanos.setNanos(999999) + checkEvaluation(UnixMillis(Literal(timestampWithNanos)), 1000L) + } + + test("UNIX_MICROS") { + var timestamp = Literal(new Timestamp(0L)) + checkEvaluation(UnixMicros(timestamp), 0L) + timestamp = Literal(new Timestamp(1000L)) + checkEvaluation(UnixMicros(timestamp), 1000000L) + // Truncates higher levels of precision + val timestampWithNanos = new Timestamp(1000L) + timestampWithNanos.setNanos(999999) + checkEvaluation(UnixMicros(Literal(timestampWithNanos)), 1000999L) + } + test("TIMESTAMP_SECONDS") { def testIntegralFunc(value: Number): Unit = { checkEvaluation( diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 0a54dff3a1cea..5f09ba2093b4e 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -1,6 +1,6 @@ ## Summary - - Number of queries: 342 + - Number of queries: 345 - Number of expressions that missing example: 13 - Expressions missing examples: bigint,binary,boolean,date,decimal,double,float,int,smallint,string,timestamp,tinyint,window ## Schema of Built-in Functions @@ -289,6 +289,9 @@ | org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct | | org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | SELECT positive(1) | struct<(+ 1):int> | | org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct | +| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01')) | struct | +| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01')) | struct | +| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01')) | struct | | org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct | | org.apache.spark.sql.catalyst.expressions.Upper | ucase | SELECT ucase('SparkSql') | struct | | org.apache.spark.sql.catalyst.expressions.Upper | upper | SELECT upper('SparkSql') | struct | diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 534e222b7c13e..5fbe3d6a33f0d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -14,6 +14,10 @@ select TIMESTAMP_MILLIS(-92233720368547758); select TIMESTAMP_SECONDS(0.1234567); -- truncation is OK for float/double select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567)); +-- UNIX_SECONDS, UNIX_MILLISECONDS and UNIX_MICROSECONDS +select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_SECONDS(null); +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null); +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null); -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index 10669f14aa87b..6a14629e1c846 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 117 +-- Number of queries: 120 -- !query @@ -87,6 +87,30 @@ struct +-- !query output +1606861808 1606861808 NULL + + +-- !query +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null) +-- !query schema +struct +-- !query output +1606861808000 1606861808999 NULL + + +-- !query +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null) +-- !query schema +struct +-- !query output +1606861808000000 1606861808999999 NULL + + -- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index 810ab6ef0cbfc..196911cbe9dfa 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 117 +-- Number of queries: 120 -- !query @@ -87,6 +87,30 @@ struct +-- !query output +1606861808 1606861808 NULL + + +-- !query +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null) +-- !query schema +struct +-- !query output +1606861808000 1606861808999 NULL + + +-- !query +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null) +-- !query schema +struct +-- !query output +1606861808000000 1606861808999999 NULL + + -- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema From 0b31348794415313a128c3bfe173ac9f72d4cf17 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 2 Dec 2020 16:10:26 +0800 Subject: [PATCH 2/9] revise test case name --- .../spark/sql/catalyst/expressions/DateExpressionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index ca5f4ed987d6b..8659ccf6cf85a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1207,7 +1207,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(UnixSeconds(timestamp), 1L) } - test("UNIX_MILLS") { + test("UNIX_MILLIS") { var timestamp = Literal(new Timestamp(0L)) checkEvaluation(UnixMillis(timestamp), 0L) timestamp = Literal(new Timestamp(1000L)) From 35ef5d2bca23c059826931dd37e4f93823c0aa37 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 2 Dec 2020 19:08:51 +0800 Subject: [PATCH 3/9] update datetime-legacy.sql.out --- .../sql-tests/results/datetime-legacy.sql.out | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out index 7c2c62a2db496..d38921300aee8 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 117 +-- Number of queries: 120 -- !query @@ -87,6 +87,30 @@ struct +-- !query output +1606861808 1606861808 NULL + + +-- !query +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null) +-- !query schema +struct +-- !query output +1606861808000 1606861808999 NULL + + +-- !query +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null) +-- !query schema +struct +-- !query output +1606861808000000 1606861808999999 NULL + + -- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema From 45495c773285b6693b8ac399f8b0ec3877baaa74 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 2 Dec 2020 20:44:07 +0800 Subject: [PATCH 4/9] address comments --- .../expressions/datetimeExpressions.scala | 6 +++--- .../expressions/DateExpressionsSuite.scala | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 294e1e4e035ea..925d2f56c25a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -534,14 +534,14 @@ abstract class TimestampToLongBase extends UnaryExpression override def dataType: DataType = LongType override def nullSafeEval(input: Any): Any = { - Math.floorDiv(input.asInstanceOf[Number].longValue(), scaleFactor) + input.asInstanceOf[Number].longValue() / scaleFactor } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { if (scaleFactor == 1) { defineCodeGen(ctx, ev, c => c) } else { - defineCodeGen(ctx, ev, c => s"java.lang.Math.floorDiv($c, ${scaleFactor}L)") + defineCodeGen(ctx, ev, c => s"$c / ${scaleFactor}L") } } } @@ -582,7 +582,7 @@ case class UnixMillis(child: Expression) extends TimestampToLongBase { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.", examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 8659ccf6cf85a..da4ec982ddc4f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1198,20 +1198,28 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } test("UNIX_SECONDS") { + checkEvaluation(UnixSeconds(Literal(null, TimestampType)), null) var timestamp = Literal(new Timestamp(0L)) checkEvaluation(UnixSeconds(timestamp), 0L) timestamp = Literal(new Timestamp(1000L)) checkEvaluation(UnixSeconds(timestamp), 1L) + timestamp = Literal(new Timestamp(-1000L)) + checkEvaluation(UnixSeconds(timestamp), -1L) // Truncates higher levels of precision timestamp = Literal(new Timestamp(1999L)) checkEvaluation(UnixSeconds(timestamp), 1L) + timestamp = Literal(new Timestamp(-1001L)) + checkEvaluation(UnixSeconds(timestamp), -1L) } test("UNIX_MILLIS") { + checkEvaluation(UnixMillis(Literal(null, TimestampType)), null) var timestamp = Literal(new Timestamp(0L)) checkEvaluation(UnixMillis(timestamp), 0L) timestamp = Literal(new Timestamp(1000L)) checkEvaluation(UnixMillis(timestamp), 1000L) + timestamp = Literal(new Timestamp(-1000L)) + checkEvaluation(UnixMillis(timestamp), -1000L) // Truncates higher levels of precision val timestampWithNanos = new Timestamp(1000L) timestampWithNanos.setNanos(999999) @@ -1219,14 +1227,16 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } test("UNIX_MICROS") { + checkEvaluation(UnixMicros(Literal(null, TimestampType)), null) var timestamp = Literal(new Timestamp(0L)) checkEvaluation(UnixMicros(timestamp), 0L) timestamp = Literal(new Timestamp(1000L)) checkEvaluation(UnixMicros(timestamp), 1000000L) - // Truncates higher levels of precision + timestamp = Literal(new Timestamp(-1000L)) + checkEvaluation(UnixMicros(timestamp), -1000000L) val timestampWithNanos = new Timestamp(1000L) - timestampWithNanos.setNanos(999999) - checkEvaluation(UnixMicros(Literal(timestampWithNanos)), 1000999L) + timestampWithNanos.setNanos(1000) // 1 microsecond + checkEvaluation(UnixMicros(Literal(timestampWithNanos)), 1000001L) } test("TIMESTAMP_SECONDS") { From 82ba97b8418e373faaab34dc6ee21014f86258f8 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 2 Dec 2020 22:06:06 +0800 Subject: [PATCH 5/9] update --- .../sql/catalyst/expressions/datetimeExpressions.scala | 4 ++-- .../sql/catalyst/expressions/DateExpressionsSuite.scala | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 925d2f56c25a5..377c227c5663d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -534,14 +534,14 @@ abstract class TimestampToLongBase extends UnaryExpression override def dataType: DataType = LongType override def nullSafeEval(input: Any): Any = { - input.asInstanceOf[Number].longValue() / scaleFactor + Math.floorDiv(input.asInstanceOf[Number].longValue(), scaleFactor) } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { if (scaleFactor == 1) { defineCodeGen(ctx, ev, c => c) } else { - defineCodeGen(ctx, ev, c => s"$c / ${scaleFactor}L") + defineCodeGen(ctx, ev, c => s"java.lang.Math.floorDiv($c, ${scaleFactor}L)") } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index da4ec982ddc4f..f99a6d4052e71 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1205,11 +1205,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(UnixSeconds(timestamp), 1L) timestamp = Literal(new Timestamp(-1000L)) checkEvaluation(UnixSeconds(timestamp), -1L) + // -1ms is considered to be in -1st second, as 0-999ms is in 0th second. + timestamp = Literal(new Timestamp(-1L)) + checkEvaluation(UnixSeconds(timestamp), -1L) + timestamp = Literal(new Timestamp(-1000L)) + checkEvaluation(UnixSeconds(timestamp), -1L) // Truncates higher levels of precision timestamp = Literal(new Timestamp(1999L)) checkEvaluation(UnixSeconds(timestamp), 1L) - timestamp = Literal(new Timestamp(-1001L)) - checkEvaluation(UnixSeconds(timestamp), -1L) } test("UNIX_MILLIS") { From f5e4b53e19975131a46e17620031631c5222e451 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 2 Dec 2020 23:15:00 +0800 Subject: [PATCH 6/9] fix test failure --- .../org/apache/spark/sql/expressions/ExpressionInfoSuite.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala index 6085c1f2cccb0..387d469a436d4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala @@ -151,6 +151,9 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession { "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp", "org.apache.spark.sql.catalyst.expressions.CurrentTimeZone", "org.apache.spark.sql.catalyst.expressions.Now", + "org.apache.spark.sql.catalyst.expressions.UnixSeconds", + "org.apache.spark.sql.catalyst.expressions.UnixMillis", + "org.apache.spark.sql.catalyst.expressions.UnixMicros", // Random output without a seed "org.apache.spark.sql.catalyst.expressions.Rand", "org.apache.spark.sql.catalyst.expressions.Randn", From 270aa9566658d82a17d1f2c0d91a1863a8d40fb2 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 3 Dec 2020 00:18:29 +0800 Subject: [PATCH 7/9] fix --- .../sql/catalyst/expressions/datetimeExpressions.scala | 6 +++--- .../apache/spark/sql/expressions/ExpressionInfoSuite.scala | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 377c227c5663d..ef85c0bf97cc2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -551,7 +551,7 @@ abstract class TimestampToLongBase extends UnaryExpression usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", examples = """ Examples: - > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01-00')); 1 """, group = "datetime_funcs", @@ -568,7 +568,7 @@ case class UnixSeconds(child: Expression) extends TimestampToLongBase { usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", examples = """ Examples: - > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01-00')); 1000 """, group = "datetime_funcs", @@ -585,7 +585,7 @@ case class UnixMillis(child: Expression) extends TimestampToLongBase { usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.", examples = """ Examples: - > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01')); + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01-00')); 1000000 """, group = "datetime_funcs", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala index 387d469a436d4..6085c1f2cccb0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala @@ -151,9 +151,6 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession { "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp", "org.apache.spark.sql.catalyst.expressions.CurrentTimeZone", "org.apache.spark.sql.catalyst.expressions.Now", - "org.apache.spark.sql.catalyst.expressions.UnixSeconds", - "org.apache.spark.sql.catalyst.expressions.UnixMillis", - "org.apache.spark.sql.catalyst.expressions.UnixMicros", // Random output without a seed "org.apache.spark.sql.catalyst.expressions.Rand", "org.apache.spark.sql.catalyst.expressions.Randn", From 46c8651ff59272d45f0b6bc24825abb1a8c3f7c2 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 3 Dec 2020 00:22:59 +0800 Subject: [PATCH 8/9] fix --- .../sql/catalyst/expressions/datetimeExpressions.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index ef85c0bf97cc2..2e97590b4145f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -551,7 +551,7 @@ abstract class TimestampToLongBase extends UnaryExpression usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", examples = """ Examples: - > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01-00')); + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); 1 """, group = "datetime_funcs", @@ -568,7 +568,7 @@ case class UnixSeconds(child: Expression) extends TimestampToLongBase { usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", examples = """ Examples: - > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01-00')); + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); 1000 """, group = "datetime_funcs", @@ -585,7 +585,7 @@ case class UnixMillis(child: Expression) extends TimestampToLongBase { usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.", examples = """ Examples: - > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01-00')); + > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); 1000000 """, group = "datetime_funcs", From 025fdd2898338c206920811787b109128acf9184 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 3 Dec 2020 02:02:02 +0800 Subject: [PATCH 9/9] address comment --- .../sql-functions/sql-expression-schema.md | 6 +++--- .../resources/sql-tests/inputs/datetime.sql | 6 +++--- .../sql-tests/results/ansi/datetime.sql.out | 18 +++++++++--------- .../sql-tests/results/datetime-legacy.sql.out | 18 +++++++++--------- .../sql-tests/results/datetime.sql.out | 18 +++++++++--------- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 5f09ba2093b4e..861062a1f7705 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -289,9 +289,9 @@ | org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct | | org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | SELECT positive(1) | struct<(+ 1):int> | | org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct | -| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01')) | struct | -| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01')) | struct | -| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01')) | struct | +| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')) | struct | +| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')) | struct | +| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')) | struct | | org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct | | org.apache.spark.sql.catalyst.expressions.Upper | ucase | SELECT ucase('SparkSql') | struct | | org.apache.spark.sql.catalyst.expressions.Upper | upper | SELECT upper('SparkSql') | struct | diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 5fbe3d6a33f0d..c2ccb3ee0db06 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -15,9 +15,9 @@ select TIMESTAMP_SECONDS(0.1234567); -- truncation is OK for float/double select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567)); -- UNIX_SECONDS, UNIX_MILLISECONDS and UNIX_MICROSECONDS -select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_SECONDS(null); -select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null); -select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null); +select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_SECONDS(null); +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null); +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null); -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index 6a14629e1c846..9d99d3b870b3f 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -88,27 +88,27 @@ struct +struct -- !query output -1606861808 1606861808 NULL +1606833008 1606833008 NULL -- !query -select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null) +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null) -- !query schema -struct +struct -- !query output -1606861808000 1606861808999 NULL +1606833008000 1606833008999 NULL -- !query -select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null) +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null) -- !query schema -struct +struct -- !query output -1606861808000000 1606861808999999 NULL +1606833008000000 1606833008999999 NULL -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out index d38921300aee8..73e9823d96a73 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out @@ -88,27 +88,27 @@ struct +struct -- !query output -1606861808 1606861808 NULL +1606833008 1606833008 NULL -- !query -select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null) +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null) -- !query schema -struct +struct -- !query output -1606861808000 1606861808999 NULL +1606833008000 1606833008999 NULL -- !query -select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null) +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null) -- !query schema -struct +struct -- !query output -1606861808000000 1606861808999999 NULL +1606833008000000 1606833008999999 NULL -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index 196911cbe9dfa..2c39c1291aa70 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -88,27 +88,27 @@ struct +struct -- !query output -1606861808 1606861808 NULL +1606833008 1606833008 NULL -- !query -select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null) +select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MILLIS(null) -- !query schema -struct +struct -- !query output -1606861808000 1606861808999 NULL +1606833008000 1606833008999 NULL -- !query -select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null) +select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08Z')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999Z')), UNIX_MICROS(null) -- !query schema -struct +struct -- !query output -1606861808000000 1606861808999999 NULL +1606833008000000 1606833008999999 NULL -- !query