From 2865e456c3a6aa78fc05ea8329f0cc10999218e8 Mon Sep 17 00:00:00 2001 From: GuoPhilipse Date: Tue, 30 Jun 2020 00:48:02 +0800 Subject: [PATCH 1/3] fix error messages --- .../sql/catalyst/analysis/CheckAnalysis.scala | 3 +- .../sql/catalyst/analysis/AnalysisSuite.scala | 73 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 9e325d0c2e4e1..9c99acaa994b8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -337,7 +337,8 @@ trait CheckAnalysis extends PredicateHelper { def ordinalNumber(i: Int): String = i match { case 0 => "first" case 1 => "second" - case i => s"${i}th" + case 2 => "third" + case i => s"${i + 1}th" } val ref = dataTypes(operator.children.head) operator.children.tail.zipWithIndex.foreach { case (child, ti) => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 189152374b0d1..77d8f8a61fc09 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -831,4 +831,77 @@ class AnalysisSuite extends AnalysisTest with Matchers { } } } + + test("SPARK-32131 Fix wrong column index when we have more than two columns" + + " during union and set operations" ) { + val firstTable = LocalRelation( + AttributeReference("a", StringType)(), + AttributeReference("b", DoubleType)(), + AttributeReference("c", IntegerType)(), + AttributeReference("d", FloatType)()) + + val secondTable = LocalRelation( + AttributeReference("a", StringType)(), + AttributeReference("b", TimestampType)(), + AttributeReference("c", IntegerType)(), + AttributeReference("d", FloatType)()) + + val thirdTable = LocalRelation( + AttributeReference("a", StringType)(), + AttributeReference("b", DoubleType)(), + AttributeReference("c", TimestampType)(), + AttributeReference("d", FloatType)()) + + val fourthTable = LocalRelation( + AttributeReference("a", StringType)(), + AttributeReference("b", DoubleType)(), + AttributeReference("c", IntegerType)(), + AttributeReference("d", TimestampType)()) + + val a1 = firstTable.output(0) + val b1 = firstTable.output(1) + val c1 = firstTable.output(2) + val d1 = firstTable.output(3) + + val a2 = secondTable.output(0) + val b2 = secondTable.output(1) + val c2 = secondTable.output(2) + val d2 = secondTable.output(3) + + val a3 = thirdTable.output(0) + val b3 = thirdTable.output(1) + val c3 = thirdTable.output(2) + val d3 = thirdTable.output(3) + + val a4 = fourthTable.output(0) + val b4 = fourthTable.output(1) + val c4 = fourthTable.output(2) + val d4 = fourthTable.output(3) + + val r1 = Union(firstTable, secondTable) + val r2 = Union(firstTable, thirdTable) + val r3 = Union(firstTable, fourthTable) + val r4 = Except(firstTable, secondTable, isAll = false) + val r5 = Intersect(firstTable, secondTable, isAll = false) + + assertAnalysisError(r1, + Seq("Union can only be performed on tables with the compatible column types. " + + "timestamp <> double at the second column of the second table")) + + assertAnalysisError(r2, + Seq("Union can only be performed on tables with the compatible column types. " + + "timestamp <> int at the third column of the second table")) + + assertAnalysisError(r3, + Seq("Union can only be performed on tables with the compatible column types. " + + "timestamp <> float at the 4th column of the second table")) + + assertAnalysisError(r4, + Seq("Except can only be performed on tables with the compatible column types. " + + "timestamp <> double at the second column of the second table")) + + assertAnalysisError(r5, + Seq("Intersect can only be performed on tables with the compatible column types. " + + "timestamp <> double at the second column of the second table")) + } } From 1aa614d2606f443cbd8de8483f7c30d11c1b50bb Mon Sep 17 00:00:00 2001 From: GuoPhilipse Date: Tue, 30 Jun 2020 14:50:24 +0800 Subject: [PATCH 2/3] remove useless variable --- .../sql/catalyst/analysis/AnalysisSuite.scala | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 77d8f8a61fc09..0ad8197b3d069 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -858,26 +858,6 @@ class AnalysisSuite extends AnalysisTest with Matchers { AttributeReference("c", IntegerType)(), AttributeReference("d", TimestampType)()) - val a1 = firstTable.output(0) - val b1 = firstTable.output(1) - val c1 = firstTable.output(2) - val d1 = firstTable.output(3) - - val a2 = secondTable.output(0) - val b2 = secondTable.output(1) - val c2 = secondTable.output(2) - val d2 = secondTable.output(3) - - val a3 = thirdTable.output(0) - val b3 = thirdTable.output(1) - val c3 = thirdTable.output(2) - val d3 = thirdTable.output(3) - - val a4 = fourthTable.output(0) - val b4 = fourthTable.output(1) - val c4 = fourthTable.output(2) - val d4 = fourthTable.output(3) - val r1 = Union(firstTable, secondTable) val r2 = Union(firstTable, thirdTable) val r3 = Union(firstTable, fourthTable) From 984c65275bdf68bf8ff1f9ce3c91696a90e65439 Mon Sep 17 00:00:00 2001 From: GuoPhilipse Date: Tue, 30 Jun 2020 16:24:01 +0800 Subject: [PATCH 3/3] improve test style --- .../org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 0ad8197b3d069..c15ec49e14282 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -832,7 +832,7 @@ class AnalysisSuite extends AnalysisTest with Matchers { } } - test("SPARK-32131 Fix wrong column index when we have more than two columns" + + test("SPARK-32131: Fix wrong column index when we have more than two columns" + " during union and set operations" ) { val firstTable = LocalRelation( AttributeReference("a", StringType)(),