diff --git a/sql/core/src/test/resources/sql-tests/results/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/limit.sql.out index 281326e22a97a..074e7a6d28c47 100644 --- a/sql/core/src/test/resources/sql-tests/results/limit.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/limit.sql.out @@ -88,7 +88,7 @@ SELECT * FROM testdata LIMIT key > 3 struct<> -- !query output org.apache.spark.sql.AnalysisException -The limit expression must evaluate to a constant value, but got (testdata.`key` > 3); +The limit expression must evaluate to a constant value, but got (spark_catalog.default.testdata.`key` > 3); -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index dd2e7b168698e..a95b02c7f7743 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -63,15 +63,9 @@ SHOW TABLES -- !query schema struct -- !query output -aggtest -arraydata -mapdata -onek show_t1 show_t2 show_t3 -tenk1 -testdata -- !query @@ -79,15 +73,9 @@ SHOW TABLES IN showdb -- !query schema struct -- !query output -aggtest -arraydata -mapdata -onek show_t1 show_t2 show_t3 -tenk1 -testdata -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out index ee8a7474e6d5d..d88790d8b5ec8 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out @@ -63,12 +63,6 @@ SHOW VIEWS -- !query schema struct -- !query output -aggtest -arraydata -mapdata -onek -tenk1 -testdata view_1 view_2 view_4 @@ -79,12 +73,6 @@ SHOW VIEWS FROM showdb -- !query schema struct -- !query output -aggtest -arraydata -mapdata -onek -tenk1 -testdata view_1 view_2 view_4 @@ -95,12 +83,6 @@ SHOW VIEWS IN showdb -- !query schema struct -- !query output -aggtest -arraydata -mapdata -onek -tenk1 -testdata view_1 view_2 view_4 @@ -111,12 +93,6 @@ SHOW VIEWS IN global_temp -- !query schema struct -- !query output -aggtest -arraydata -mapdata -onek -tenk1 -testdata view_3 view_4 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index cb11f9bb9fe96..bc9b4d85aacdb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -363,7 +363,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { // Create a local SparkSession to have stronger isolation between different test cases. // This does not isolate catalog changes. val localSparkSession = spark.newSession() - loadTestData(localSparkSession) testCase match { case udfTestCase: UDFTest => @@ -572,14 +571,20 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { } /** Load built-in test tables into the SparkSession. */ - private def loadTestData(session: SparkSession): Unit = { + private def createTestTables(session: SparkSession): Unit = { import session.implicits._ - (1 to 100).map(i => (i, i.toString)).toDF("key", "value").createOrReplaceTempView("testdata") + (1 to 100).map(i => (i, i.toString)).toDF("key", "value") + .repartition(1) + .write + .format("parquet") + .saveAsTable("testdata") ((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4))) :: Nil) .toDF("arraycol", "nestedarraycol") - .createOrReplaceTempView("arraydata") + .write + .format("parquet") + .saveAsTable("arraydata") (Tuple1(Map(1 -> "a1", 2 -> "b1", 3 -> "c1", 4 -> "d1", 5 -> "e1")) :: Tuple1(Map(1 -> "a2", 2 -> "b2", 3 -> "c2", 4 -> "d2")) :: @@ -587,7 +592,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { Tuple1(Map(1 -> "a4", 2 -> "b4")) :: Tuple1(Map(1 -> "a5")) :: Nil) .toDF("mapcol") - .createOrReplaceTempView("mapdata") + .write + .format("parquet") + .saveAsTable("mapdata") session .read @@ -595,7 +602,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { .options(Map("delimiter" -> "\t", "header" -> "false")) .schema("a int, b float") .load(testFile("test-data/postgresql/agg.data")) - .createOrReplaceTempView("aggtest") + .write + .format("parquet") + .saveAsTable("aggtest") session .read @@ -621,7 +630,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { |string4 string """.stripMargin) .load(testFile("test-data/postgresql/onek.data")) - .createOrReplaceTempView("onek") + .write + .format("parquet") + .saveAsTable("onek") session .read @@ -647,7 +658,18 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { |string4 string """.stripMargin) .load(testFile("test-data/postgresql/tenk.data")) - .createOrReplaceTempView("tenk1") + .write + .format("parquet") + .saveAsTable("tenk1") + } + + private def removeTestTables(session: SparkSession): Unit = { + session.sql("DROP TABLE IF EXISTS testdata") + session.sql("DROP TABLE IF EXISTS arraydata") + session.sql("DROP TABLE IF EXISTS mapdata") + session.sql("DROP TABLE IF EXISTS aggtest") + session.sql("DROP TABLE IF EXISTS onek") + session.sql("DROP TABLE IF EXISTS tenk1") } private val originalTimeZone = TimeZone.getDefault @@ -655,6 +677,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { override def beforeAll(): Unit = { super.beforeAll() + createTestTables(spark) // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) // Add Locale setting @@ -668,6 +691,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { try { TimeZone.setDefault(originalTimeZone) Locale.setDefault(originalLocale) + removeTestTables(spark) // For debugging dump some statistics about how much time was spent in various optimizer rules logWarning(RuleExecutor.dumpTimeSpent())