diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 48a9f57b75dd4..4de42c0a987a1 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -8049,6 +8049,11 @@ " is a VARIABLE and cannot be updated using the SET statement. Use SET VARIABLE = ... instead." ] }, + "SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION" : { + "message" : [ + "SHOW TABLE EXTENDED with PARTITION does not support AS JSON output." + ] + }, "SQL_CURSOR" : { "message" : [ "SQL cursor operations (DECLARE CURSOR, OPEN, FETCH, CLOSE) are not supported." diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 735921681cdcd..9f125d4d4e8f0 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -360,9 +360,9 @@ statement | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? (statement|setResetStatement) #explain | SHOW TABLES ((FROM | IN) identifierReference)? - (LIKE? pattern=stringLit)? #showTables + (LIKE? pattern=stringLit)? (AS JSON)? #showTables | SHOW TABLE EXTENDED ((FROM | IN) ns=identifierReference)? - LIKE pattern=stringLit partitionSpec? #showTableExtended + LIKE pattern=stringLit partitionSpec? (AS JSON)? #showTableExtended | SHOW TBLPROPERTIES table=identifierReference (LEFT_PAREN key=propertyKeyOrStringLit RIGHT_PAREN)? #showTblProperties | SHOW COLUMNS (FROM | IN) table=identifierReference diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala index 6a275b9ad0c16..8ddf82225ca54 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala @@ -53,6 +53,12 @@ private[sql] trait CompilationErrors extends DataTypeErrorsBase { messageParameters = Map.empty) } + def showTableExtendedJsonWithPartitionError(): AnalysisException = { + new AnalysisException( + errorClass = "UNSUPPORTED_FEATURE.SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION", + messageParameters = Map.empty) + } + def cannotFindDescriptorFileError(filePath: String, cause: Throwable): AnalysisException = { new AnalysisException( errorClass = "PROTOBUF_DESCRIPTOR_FILE_NOT_FOUND", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 9db80f894da73..d1d65b64d24b6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -5795,7 +5795,10 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - ShowTables(ns, Option(ctx.pattern).map(x => string(visitStringLit(x)))) + val asJson = ctx.JSON != null + val pattern = Option(ctx.pattern).map(x => string(visitStringLit(x))) + val output = if (asJson) ShowTables.getJsonOutputAttrs else ShowTables.getOutputAttrs + ShowTables(ns, pattern, asJson, output) } /** @@ -5803,6 +5806,10 @@ class AstBuilder extends DataTypeAstBuilder */ override def visitShowTableExtended( ctx: ShowTableExtendedContext): LogicalPlan = withOrigin(ctx) { + val asJson = ctx.JSON != null + if (asJson && ctx.partitionSpec != null) { + throw QueryCompilationErrors.showTableExtendedJsonWithPartitionError() + } Option(ctx.partitionSpec).map { spec => val table = withOrigin(ctx.pattern) { if (ctx.identifierReference() != null) { @@ -5822,7 +5829,8 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - ShowTablesExtended(ns, string(visitStringLit(ctx.pattern))) + val output = if (asJson) ShowTables.getJsonOutputAttrs else ShowTablesUtils.getOutputAttrs + ShowTablesExtended(ns, string(visitStringLit(ctx.pattern)), asJson, output) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index 7b657ce34df45..e31c542dbcb97 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -1371,10 +1371,12 @@ case class RenameTable( case class ShowTables( namespace: LogicalPlan, pattern: Option[String], + asJson: Boolean = false, override val output: Seq[Attribute] = ShowTables.getOutputAttrs) extends UnaryCommand { override def child: LogicalPlan = namespace override protected def withNewChildInternal(newChild: LogicalPlan): ShowTables = copy(namespace = newChild) + override protected def stringArgs: Iterator[Any] = Iterator(pattern, output) } object ShowTables { @@ -1382,6 +1384,9 @@ object ShowTables { AttributeReference("namespace", StringType, nullable = false)(), AttributeReference("tableName", StringType, nullable = false)(), AttributeReference("isTemporary", BooleanType, nullable = false)()) + + def getJsonOutputAttrs: Seq[Attribute] = Seq( + AttributeReference("json_metadata", StringType, nullable = false)()) } /** @@ -1390,10 +1395,12 @@ object ShowTables { case class ShowTablesExtended( namespace: LogicalPlan, pattern: String, + asJson: Boolean = false, override val output: Seq[Attribute] = ShowTablesUtils.getOutputAttrs) extends UnaryCommand { override def child: LogicalPlan = namespace override protected def withNewChildInternal(newChild: LogicalPlan): ShowTablesExtended = copy(namespace = newChild) + override protected def stringArgs: Iterator[Any] = Iterator(pattern, output) } object ShowTablesUtils { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index 8f0b664e10c5b..9d6e9339e821f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -366,19 +366,23 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) case d @ DropNamespace(ResolvedV1Database(db), _, _) if conf.useV1Command => DropDatabaseCommand(db, d.ifExists, d.cascade) - case ShowTables(ResolvedV1Database(db), pattern, output) if conf.useV1Command => - ShowTablesCommand(Some(db), pattern, output) + case ShowTables(ResolvedV1Database(db), pattern, asJson, output) if conf.useV1Command => + ShowTablesCommand(Some(db), pattern, output, asJson = asJson) case ShowTablesExtended( ResolvedV1Database(db), pattern, + asJson, output) => - val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { + val newOutput = if (asJson) { + output + } else if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { output.head.withName("database") +: output.tail } else { output } - ShowTablesCommand(Some(db), Some(pattern), newOutput, isExtended = true) + ShowTablesCommand(Some(db), Some(pattern), newOutput, + isExtended = true, asJson = asJson) case ShowTablePartition( ResolvedTable(catalog, _, table: V1Table, _), diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 40c40f6ea78aa..939b12a404fd2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -195,7 +195,7 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog with Logging { private def makeTablesDataset(plan: ShowTables): Dataset[Table] = { val qe = sparkSession.sessionState.executePlan(plan) val catalog = qe.analyzed.collectFirst { - case ShowTables(r: ResolvedNamespace, _, _) => r.catalog + case ShowTables(r: ResolvedNamespace, _, _, _) => r.catalog case _: ShowTablesCommand => sparkSession.sessionState.catalogManager.v2SessionCatalog }.get diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index ca534706635a1..319f3284c7c66 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -25,6 +25,8 @@ import scala.util.control.NonFatal import org.apache.hadoop.fs.{FileContext, FsConstants, Path} import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission} +import org.json4s._ +import org.json4s.jackson.JsonMethods._ import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.{SQLConfHelper, TableIdentifier} @@ -927,9 +929,21 @@ case class ShowTablesCommand( tableIdentifierPattern: Option[String], override val output: Seq[Attribute], isExtended: Boolean = false, - partitionSpec: Option[TablePartitionSpec] = None) extends LeafRunnableCommand { + partitionSpec: Option[TablePartitionSpec] = None, + asJson: Boolean = false) extends LeafRunnableCommand { + + override protected def stringArgs: Iterator[Any] = + Iterator(databaseName, tableIdentifierPattern, output, isExtended, partitionSpec) override def run(sparkSession: SparkSession): Seq[Row] = { + if (asJson) { + runAsJson(sparkSession) + } else { + runAsText(sparkSession) + } + } + + private def runAsText(sparkSession: SparkSession): Seq[Row] = { // Since we need to return a Seq of rows, we will call getTables directly // instead of calling tables in sparkSession. val catalog = sparkSession.sessionState.catalog @@ -972,6 +986,45 @@ case class ShowTablesCommand( Seq(Row(database, tableName, isTemp, s"$information\n")) } } + + private def runAsJson(sparkSession: SparkSession): Seq[Row] = { + val catalog = sparkSession.sessionState.catalog + val db = databaseName.getOrElse(catalog.getCurrentDatabase) + val tables = + tableIdentifierPattern.map(catalog.listTables(db, _)).getOrElse(catalog.listTables(db)) + + val jsonTables = tables.map { tableIdent => + val isTemp = catalog.isTempView(tableIdent) + val ns = tableIdent.database.toList + + if (isExtended) { + val tableType = if (isTemp) { + "VIEW" + } else { + val meta = catalog.getTempViewOrPermanentTableMetadata(tableIdent) + if (meta.tableType == CatalogTableType.VIEW) "VIEW" else "TABLE" + } + + JObject( + "name" -> JString(tableIdent.table), + "catalog" -> JString( + sparkSession.sessionState.catalogManager.v2SessionCatalog.name()), + "namespace" -> JArray(ns.map(JString(_))), + "type" -> JString(tableType), + "isTemporary" -> JBool(isTemp) + ) + } else { + JObject( + "name" -> JString(tableIdent.table), + "namespace" -> JArray(ns.map(JString(_))), + "isTemporary" -> JBool(isTemp) + ) + } + }.toList + + val jsonOutput = JObject("tables" -> JArray(jsonTables)) + Seq(Row(compact(render(jsonOutput)))) + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index e113475811092..1638ae66f9ba6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -658,8 +658,13 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case DropNamespace(ResolvedNamespace(catalog, ns, _), ifExists, cascade) => DropNamespaceExec(catalog, ns, ifExists, cascade) :: Nil - case ShowTables(ResolvedNamespace(catalog, ns, _), pattern, output) => - ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + case ShowTables(ResolvedNamespace(catalog, ns, _), pattern, asJson, output) => + if (asJson) { + ShowTablesJsonExec( + output, catalog.asTableCatalog, ns, pattern.getOrElse("*"), isExtended = false) :: Nil + } else { + ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + } // SHOW VIEWS on a v2 ViewCatalog. `ResolveSessionCatalog` rewrites the SHOW VIEWS plan to // v1 `ShowViewsCommand` only when the catalog is NOT a `ViewCatalog`; non-`ViewCatalog` @@ -673,8 +678,14 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case ShowTablesExtended( ResolvedNamespace(catalog, ns, _), pattern, + asJson, output) => - ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + if (asJson) { + ShowTablesJsonExec( + output, catalog.asTableCatalog, ns, pattern, isExtended = true) :: Nil + } else { + ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + } case ShowTablePartition(r: ResolvedTable, part, output) => ShowTablePartitionExec(output, r.catalog, r.identifier, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala index 8680785e0815f..c74d9f8fa748a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala @@ -41,19 +41,19 @@ case class ShowTablesExec( namespace: Seq[String], pattern: Option[String]) extends V2CommandExec with LeafExecNode { override protected def run(): Seq[InternalRow] = { - val rows = new ArrayBuffer[InternalRow]() - val identifiers: Array[Identifier] = catalog match { case mc: TableViewCatalog => mc.listTableAndViewSummaries(namespace.toArray).map(_.identifier()) case _ => catalog.listTables(namespace.toArray) } - identifiers.foreach { ident => - if (pattern.map(StringUtils.filterPattern(Seq(ident.name()), _).nonEmpty).getOrElse(true)) { - rows += toCatalystRow(ident.namespace().quoted, ident.name(), isTempView(ident, catalog)) - } + val filteredIdents = identifiers.filter { ident => + pattern.map(StringUtils.filterPattern(Seq(ident.name()), _).nonEmpty).getOrElse(true) } + val rows = new ArrayBuffer[InternalRow]() + filteredIdents.foreach { ident => + rows += toCatalystRow(ident.namespace().quoted, ident.name(), isTempView(ident, catalog)) + } rows.toSeq } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala new file mode 100644 index 0000000000000..8af65d76c344b --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2 + +import scala.collection.mutable.ArrayBuffer + +import org.json4s._ +import org.json4s.jackson.JsonMethods._ + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.util.StringUtils +import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, TableCatalog, TableViewCatalog} +import org.apache.spark.sql.execution.LeafExecNode + +/** + * Physical plan node for `SHOW TABLES AS JSON` and `SHOW TABLE EXTENDED AS JSON`. + * + * For a [[TableViewCatalog]] (non-extended only), listing is done via + * [[TableViewCatalog#listTableAndViewSummaries]] so that views appear alongside tables, + * matching the v1 `SHOW TABLES` semantics. + */ +case class ShowTablesJsonExec( + output: Seq[Attribute], + catalog: TableCatalog, + namespace: Seq[String], + pattern: String, + isExtended: Boolean) extends V2CommandExec with LeafExecNode { + + override protected def run(): Seq[InternalRow] = { + val identifiers: Array[Identifier] = if (!isExtended) { + catalog match { + case mc: TableViewCatalog => + mc.listTableAndViewSummaries(namespace.toArray).map(_.identifier()) + case _ => catalog.listTables(namespace.toArray) + } + } else { + catalog.listTables(namespace.toArray) + } + + val filteredIdents = identifiers.filter { ident => + StringUtils.filterPattern(Seq(ident.name()), pattern).nonEmpty + } + + val jsonRows = new ArrayBuffer[JObject]() + filteredIdents.foreach { ident => + jsonRows += toJsonEntry(ident.name(), ident.namespace(), isTempView(ident)) + } + + // For non-session V2 catalogs that don't surface temp views via listTables() or + // listTableAndViewSummaries(), fetch them separately. For V2SessionCatalog, + // listTables() already includes local temp views, so we skip this to avoid duplicates. + // For TableViewCatalog (non-extended path), views come from listTableAndViewSummaries(). + if (!CatalogV2Util.isSessionCatalog(catalog) && + (isExtended || !catalog.isInstanceOf[TableViewCatalog])) { + val sessionCatalog = session.sessionState.catalog + val db = namespace match { + case Seq(db) => db + case _ => "" + } + sessionCatalog.listTempViews(db, pattern).foreach { tempView => + jsonRows += toJsonEntry( + tempView.identifier.table, + tempView.identifier.database.toArray, + isTemporary = true) + } + } + + val jsonOutput = JObject("tables" -> JArray(jsonRows.toList)) + Seq(toCatalystRow(compact(render(jsonOutput)))) + } + + private def toJsonEntry( + name: String, + namespace: Array[String], + isTemporary: Boolean): JObject = { + val nsArray = JArray(namespace.map(JString(_)).toList) + if (isExtended) { + JObject( + "name" -> JString(name), + "catalog" -> JString(catalog.name()), + "namespace" -> nsArray, + "type" -> JString(if (isTemporary) "VIEW" else "TABLE"), + "isTemporary" -> JBool(isTemporary) + ) + } else { + JObject( + "name" -> JString(name), + "namespace" -> nsArray, + "isTemporary" -> JBool(isTemporary) + ) + } + } + + private def isTempView(ident: Identifier): Boolean = { + if (CatalogV2Util.isSessionCatalog(catalog)) { + session.sessionState.catalog.isTempView((ident.namespace() :+ ident.name()).toSeq) + } else false + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala index 2a4a49c75ad92..fd883c67389a9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution.command +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, CurrentNamespace, UnresolvedNamespace, UnresolvedPartitionSpec, UnresolvedTable} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan import org.apache.spark.sql.catalyst.plans.logical.{ShowTablePartition, ShowTables, ShowTablesExtended} @@ -49,6 +50,21 @@ class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*"))) } + test("show tables as json") { + comparePlans( + parsePlan("SHOW TABLES AS JSON"), + ShowTables(CurrentNamespace, None, asJson = true, + output = ShowTables.getJsonOutputAttrs)) + comparePlans( + parsePlan("SHOW TABLES IN ns1 AS JSON"), + ShowTables(UnresolvedNamespace(Seq("ns1")), None, asJson = true, + output = ShowTables.getJsonOutputAttrs)) + comparePlans( + parsePlan("SHOW TABLES IN ns1 LIKE '*test*' AS JSON"), + ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*"), asJson = true, + output = ShowTables.getJsonOutputAttrs)) + } + test("show table extended") { comparePlans( parsePlan("SHOW TABLE EXTENDED LIKE '*test*'"), @@ -80,4 +96,25 @@ class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { "SHOW TABLE EXTENDED ... PARTITION ..."), UnresolvedPartitionSpec(Map("ds" -> "2008-04-09")))) } + + test("show table extended as json") { + comparePlans( + parsePlan("SHOW TABLE EXTENDED LIKE '*test*' AS JSON"), + ShowTablesExtended(CurrentNamespace, "*test*", asJson = true, + output = ShowTables.getJsonOutputAttrs)) + comparePlans( + parsePlan(s"SHOW TABLE EXTENDED IN $catalog.ns1.ns2 LIKE '*test*' AS JSON"), + ShowTablesExtended(UnresolvedNamespace(Seq(catalog, "ns1", "ns2")), "*test*", + asJson = true, output = ShowTables.getJsonOutputAttrs)) + } + + test("show table extended as json with partition should fail") { + checkError( + exception = intercept[AnalysisException] { + parsePlan("SHOW TABLE EXTENDED LIKE '*test*' PARTITION(ds='2008-04-09') AS JSON") + }, + condition = "UNSUPPORTED_FEATURE.SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION", + parameters = Map.empty + ) + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala index dbeb67c253208..92816793604bd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.execution.command +import org.json4s._ +import org.json4s.jackson.JsonMethods._ + import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.internal.SQLConf @@ -32,6 +35,8 @@ import org.apache.spark.sql.internal.SQLConf * - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowTablesSuite` */ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { + implicit val formats: Formats = DefaultFormats + override val command = "SHOW TABLES" protected def defaultNamespace: Seq[String] @@ -461,4 +466,190 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } } + + test("SHOW TABLES AS JSON returns single row with json_metadata column") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT, data STRING) $defaultUsing") + val df = sql(s"SHOW TABLES IN $catalog.ns AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1, s"Expected 1 entry, got: $tables") + val tblEntry = tables.find(t => (t \ "name").extract[String] == "tbl") + assert(tblEntry.isDefined) + assert((tblEntry.get \ "isTemporary").extract[Boolean] == false) + assert((tblEntry.get \ "namespace").isInstanceOf[JArray]) + } + } + + test("SHOW TABLES AS JSON with empty database") { + withNamespace(s"$catalog.ns_empty") { + sql(s"CREATE NAMESPACE $catalog.ns_empty") + val df = sql(s"SHOW TABLES IN $catalog.ns_empty AS JSON") + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.isEmpty) + } + } + + test("SHOW TABLE EXTENDED AS JSON returns single row with json_metadata column") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT, data STRING) $defaultUsing") + val df = sql(s"SHOW TABLE EXTENDED IN $catalog.ns LIKE 'tbl' AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1) + val entry = tables.head + assert((entry \ "name").extract[String] == "tbl") + assert((entry \ "type").extract[String] == "TABLE") + assert((entry \ "isTemporary").extract[Boolean] == false) + assert((entry \ "catalog").isInstanceOf[JString]) + assert((entry \ "namespace").isInstanceOf[JArray]) + } + } + + test("SHOW TABLES AS JSON includes temp views") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + withTempView("tv") { + sql("CREATE TEMP VIEW tv AS SELECT 1 AS id") + val df = sql(s"SHOW TABLES IN $catalog.ns AS JSON") + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + // Verify no duplicate entries + val names = tables.map(t => (t \ "name").extract[String]) + assert(names.distinct.length == names.length, s"Duplicate entries found: $names") + val tempView = tables.find(t => (t \ "name").extract[String] == "tv") + assert(tempView.isDefined) + assert((tempView.get \ "isTemporary").extract[Boolean] == true) + } + } + } + + test("SHOW TABLE EXTENDED AS JSON with local temp view") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val localTmpViewName = "tbl_local_tmp" + withTempView(localTmpViewName) { + sql(s"CREATE TEMPORARY VIEW $localTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLE EXTENDED IN $catalog.ns LIKE 'tbl*' AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 2, s"Expected 2 entries (tbl + $localTmpViewName), got: $tables") + + // Verify no duplicate entries + val names = tables.map(e => (e \ "name").extract[String]) + assert(names.distinct.length == names.length, s"Duplicate entries found: $names") + + val tblEntry = tables.find(e => (e \ "name").extract[String] == "tbl") + assert(tblEntry.isDefined) + assert((tblEntry.get \ "isTemporary").extract[Boolean] == false) + assert((tblEntry.get \ "type").extract[String] == "TABLE") + + val tempViewEntry = tables.find(e => (e \ "name").extract[String] == localTmpViewName) + assert(tempViewEntry.isDefined) + assert((tempViewEntry.get \ "isTemporary").extract[Boolean] == true) + assert((tempViewEntry.get \ "type").extract[String] == "VIEW") + } + } + } + + test("SHOW TABLE EXTENDED AS JSON with global temp view") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val globalTmpViewName = "ext_json_gtv" + val globalNamespace = "global_temp" + withView(s"$globalNamespace.$globalTmpViewName") { + sql(s"CREATE OR REPLACE GLOBAL TEMP VIEW $globalTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLE EXTENDED IN $globalNamespace LIKE 'ext_json*' AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1, s"Expected 1 entry, got: $tables") + + val globalTempViewEntry = + tables.find(e => (e \ "name").extract[String] == globalTmpViewName) + assert(globalTempViewEntry.isDefined) + assert((globalTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + assert((globalTempViewEntry.get \ "type").extract[String] == "VIEW") + } + } + } + + test("SHOW TABLES AS JSON with global temp view") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val globalTmpViewName = "show_json_gtv" + val globalNamespace = "global_temp" + withView(s"$globalNamespace.$globalTmpViewName") { + sql(s"CREATE OR REPLACE GLOBAL TEMP VIEW $globalTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLES IN $globalNamespace AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1, s"Expected 1 entry, got: $tables") + + val globalTempViewEntry = + tables.find(e => (e \ "name").extract[String] == globalTmpViewName) + assert(globalTempViewEntry.isDefined) + assert((globalTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + } + } + } + + test("SHOW TABLE EXTENDED AS JSON with both local and global temp views") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val localTmpViewName = "both_json_ltv" + val globalTmpViewName = "both_json_gtv" + val globalNamespace = "global_temp" + withView(localTmpViewName, s"$globalNamespace.$globalTmpViewName") { + sql(s"CREATE OR REPLACE TEMP VIEW $localTmpViewName AS SELECT id FROM $t") + sql(s"CREATE OR REPLACE GLOBAL TEMP VIEW $globalTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLE EXTENDED IN $globalNamespace LIKE 'both_json*' AS JSON") + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + + assert(tables.length == 2) + + val globalTempViewEntry = + tables.find(e => (e \ "name").extract[String] == globalTmpViewName) + assert(globalTempViewEntry.isDefined) + assert((globalTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + + val localTempViewEntry = + tables.find(e => (e \ "name").extract[String] == localTmpViewName) + assert(localTempViewEntry.isDefined) + assert((localTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + } + } + } }