From c9d01df0ee9851751469e231069ab587975f5a7c Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 9 Nov 2018 00:07:46 +0800 Subject: [PATCH 1/3] fix --- .../spark/sql/catalyst/parser/SqlBase.g4 | 1 + .../execution/SQLWindowFunctionSuite.scala | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index e2d34d1650ddc..5e732edb17baa 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -691,6 +691,7 @@ namedWindow windowSpec : name=identifier #windowRef + | '('name=identifier')' #windowRef | '(' ( CLUSTER BY partition+=expression (',' partition+=expression)* | ((PARTITION | DISTRIBUTE) BY partition+=expression (',' partition+=expression)*)? diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala index 1c6fc3530cbe1..3d08d908c7873 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.test.SharedSQLContext case class WindowData(month: Int, area: String, product: Int) +case class EmpSalary(empno: Int, depname: String, salary: Double) /** @@ -31,6 +32,19 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSQLContext { import testImplicits._ + val empSalaryData = Seq( + EmpSalary(11, "develop", 5200D), + EmpSalary(7, "develop", 4200D), + EmpSalary(9, "develop", 4500D), + EmpSalary(8, "develop", 6000D), + EmpSalary(10, "develop", 5200D), + EmpSalary(5, "personnel", 3500D), + EmpSalary(2, "personnel", 3900D), + EmpSalary(3, "sales", 4800D), + EmpSalary(1, "sales", 5000D), + EmpSalary(4, "sales", 4800D) + ) + test("window function: udaf with aggregate expression") { val data = Seq( WindowData(1, "a", 5), @@ -173,6 +187,30 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSQLContext { ).map(i => Row(i._1, i._2, i._3, i._4))) } + test("window function: allow parentheses around window reference") { + sparkContext.parallelize(empSalaryData).toDF().createOrReplaceTempView("empsalary") + + checkAnswer( + sql( + """ + |SELECT sum(salary) OVER (w), avg(salary) OVER w + |FROM empsalary + |WINDOW w AS (PARTITION BY depname ORDER BY salary DESC) + """.stripMargin), + Seq( + (6000, 6000), + (16400, 5466.6666666666666667D), + (16400, 5466.6666666666666667), + (20900, 5225), + (25100, 5020), + (3900, 3900), + (7400, 3700), + (5000, 5000), + (14600, 4866.6666666666666667), + (14600, 4866.6666666666666667) + ).map(i => Row(i._1, i._2))) + } + test("window function: distinct should not be silently ignored") { val data = Seq( WindowData(1, "a", 5), From 2da6f998e4ee95d6cfbf2e8258c3a160220a366c Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 9 Nov 2018 09:57:47 +0800 Subject: [PATCH 2/3] use window.sql for testing --- .../resources/sql-tests/inputs/window.sql | 6 +++ .../sql-tests/results/window.sql.out | 19 +++++++++- .../execution/SQLWindowFunctionSuite.scala | 38 ------------------- 3 files changed, 24 insertions(+), 39 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql index cda4db4b449fe..86dc72beb7368 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/window.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql @@ -109,3 +109,9 @@ last_value(false, false) OVER w AS last_value_contain_null FROM testData WINDOW w AS () ORDER BY cate, val; + +-- parentheses around window reference +SELECT cate, sum(val) OVER (w) +FROM testData +WHERE val is not null +WINDOW w AS (PARTITION BY cate ORDER BY val); \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out index 5071e0bd26b2a..367dc4f513635 100644 --- a/sql/core/src/test/resources/sql-tests/results/window.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 22 +-- Number of queries: 23 -- !query 0 @@ -363,3 +363,20 @@ NULL a false true false false true false 1 b false true false false true false 2 b false true false false true false 3 b false true false false true false + + +-- !query 22 +SELECT cate, sum(val) OVER (w) +FROM testData +WHERE val is not null +WINDOW w AS (PARTITION BY cate ORDER BY val) +-- !query 22 schema +struct +-- !query 22 output +NULL 3 +a 2 +a 2 +a 4 +b 1 +b 3 +b 6 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala index 3d08d908c7873..1c6fc3530cbe1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala @@ -22,7 +22,6 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.test.SharedSQLContext case class WindowData(month: Int, area: String, product: Int) -case class EmpSalary(empno: Int, depname: String, salary: Double) /** @@ -32,19 +31,6 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSQLContext { import testImplicits._ - val empSalaryData = Seq( - EmpSalary(11, "develop", 5200D), - EmpSalary(7, "develop", 4200D), - EmpSalary(9, "develop", 4500D), - EmpSalary(8, "develop", 6000D), - EmpSalary(10, "develop", 5200D), - EmpSalary(5, "personnel", 3500D), - EmpSalary(2, "personnel", 3900D), - EmpSalary(3, "sales", 4800D), - EmpSalary(1, "sales", 5000D), - EmpSalary(4, "sales", 4800D) - ) - test("window function: udaf with aggregate expression") { val data = Seq( WindowData(1, "a", 5), @@ -187,30 +173,6 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSQLContext { ).map(i => Row(i._1, i._2, i._3, i._4))) } - test("window function: allow parentheses around window reference") { - sparkContext.parallelize(empSalaryData).toDF().createOrReplaceTempView("empsalary") - - checkAnswer( - sql( - """ - |SELECT sum(salary) OVER (w), avg(salary) OVER w - |FROM empsalary - |WINDOW w AS (PARTITION BY depname ORDER BY salary DESC) - """.stripMargin), - Seq( - (6000, 6000), - (16400, 5466.6666666666666667D), - (16400, 5466.6666666666666667), - (20900, 5225), - (25100, 5020), - (3900, 3900), - (7400, 3700), - (5000, 5000), - (14600, 4866.6666666666666667), - (14600, 4866.6666666666666667) - ).map(i => Row(i._1, i._2))) - } - test("window function: distinct should not be silently ignored") { val data = Seq( WindowData(1, "a", 5), From 471092d417666f5cf8908318aed098d6f06c4900 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 9 Nov 2018 11:58:07 +0800 Subject: [PATCH 3/3] add newline --- sql/core/src/test/resources/sql-tests/inputs/window.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql index 86dc72beb7368..faab4c61c8640 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/window.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql @@ -114,4 +114,4 @@ ORDER BY cate, val; SELECT cate, sum(val) OVER (w) FROM testData WHERE val is not null -WINDOW w AS (PARTITION BY cate ORDER BY val); \ No newline at end of file +WINDOW w AS (PARTITION BY cate ORDER BY val);