diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala index c9d050768c15e..d62148b2bbe45 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala @@ -53,18 +53,6 @@ class CatalogManager( } } - private def defaultCatalog: Option[CatalogPlugin] = { - conf.defaultV2Catalog.flatMap { catalogName => - try { - Some(catalog(catalogName)) - } catch { - case NonFatal(e) => - logError(s"Cannot load default v2 catalog: $catalogName", e) - None - } - } - } - private def loadV2SessionCatalog(): CatalogPlugin = { Catalogs.load(SESSION_CATALOG_NAME, conf) match { case extension: CatalogExtension => @@ -127,9 +115,7 @@ class CatalogManager( private var _currentCatalogName: Option[String] = None def currentCatalog: CatalogPlugin = synchronized { - _currentCatalogName.map(catalogName => catalog(catalogName)) - .orElse(defaultCatalog) - .getOrElse(v2SessionCatalog) + catalog(_currentCatalogName.getOrElse(conf.getConf(SQLConf.DEFAULT_CATALOG))) } def setCurrentCatalog(catalogName: String): Unit = synchronized { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index a228d9f064a1e..b71484fc75610 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -37,7 +37,6 @@ import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator import org.apache.spark.sql.catalyst.plans.logical.HintErrorHandler import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME -import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.unsafe.array.ByteArrayMethods import org.apache.spark.util.Utils @@ -2009,10 +2008,11 @@ object SQLConf { .booleanConf .createWithDefault(false) - val DEFAULT_V2_CATALOG = buildConf("spark.sql.default.catalog") - .doc("Name of the default v2 catalog, used when a catalog is not identified in queries") + val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog") + .doc("Name of the default catalog. This will be the current catalog if users have not " + + "explicitly set the current catalog yet.") .stringConf - .createOptional + .createWithDefault(SESSION_CATALOG_NAME) val V2_SESSION_CATALOG_IMPLEMENTATION = buildConf(s"spark.sql.catalog.$SESSION_CATALOG_NAME") @@ -2546,8 +2546,6 @@ class SQLConf extends Serializable with Logging { def castDatetimeToString: Boolean = getConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING) - def defaultV2Catalog: Option[String] = getConf(DEFAULT_V2_CATALOG) - def ignoreDataLocality: Boolean = getConf(SQLConf.IGNORE_DATA_LOCALITY) /** ********************** SQLConf functionality methods ************ */ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala index fcbf139fd97c1..fc7b2e720b7b7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogManagerSuite.scala @@ -43,7 +43,7 @@ class CatalogManagerSuite extends SparkFunSuite { assert(catalogManager.currentNamespace.sameElements(Array("default"))) conf.setConfString("spark.sql.catalog.dummy", classOf[DummyCatalog].getName) - conf.setConfString(SQLConf.DEFAULT_V2_CATALOG.key, "dummy") + conf.setConfString(SQLConf.DEFAULT_CATALOG.key, "dummy") // The current catalog should be changed if the default catalog is set. assert(catalogManager.currentCatalog.name() == "dummy") @@ -60,7 +60,7 @@ class CatalogManagerSuite extends SparkFunSuite { assert(catalogManager.currentNamespace.sameElements(Array("a", "b"))) conf.setConfString("spark.sql.catalog.dummy2", classOf[DummyCatalog].getName) - conf.setConfString(SQLConf.DEFAULT_V2_CATALOG.key, "dummy2") + conf.setConfString(SQLConf.DEFAULT_CATALOG.key, "dummy2") // The current catalog shouldn't be changed if it's set before. assert(catalogManager.currentCatalog.name() == "dummy") } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 0b507ebd89a15..eca52881bb786 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -221,7 +221,7 @@ class DataSourceV2SQLSuite } test("CreateTable: use default catalog for v2 sources when default catalog is set") { - spark.conf.set("spark.sql.default.catalog", "testcat") + spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat") spark.sql(s"CREATE TABLE table_name (id bigint, data string) USING foo") val testCatalog = catalog("testcat").asTableCatalog @@ -488,7 +488,7 @@ class DataSourceV2SQLSuite } test("CreateTableAsSelect: use default catalog for v2 sources when default catalog is set") { - spark.conf.set("spark.sql.default.catalog", "testcat") + spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat") val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data") df.createOrReplaceTempView("source") @@ -709,7 +709,7 @@ class DataSourceV2SQLSuite } test("ShowTables: namespace is not specified and default v2 catalog is set") { - spark.conf.set("spark.sql.default.catalog", "testcat") + spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat") spark.sql("CREATE TABLE testcat.table (id bigint, data string) USING foo") // v2 catalog is used where default namespace is empty for TestInMemoryTableCatalog. @@ -831,7 +831,7 @@ class DataSourceV2SQLSuite } test("ShowNamespaces: show root namespaces with default v2 catalog") { - spark.conf.set("spark.sql.default.catalog", "testcat") + spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat") testShowNamespaces("SHOW NAMESPACES", Seq()) @@ -874,7 +874,7 @@ class DataSourceV2SQLSuite spark.conf.set( "spark.sql.catalog.testcat_no_namspace", classOf[BasicInMemoryTableCatalog].getName) - spark.conf.set("spark.sql.default.catalog", "testcat_no_namspace") + spark.conf.set(SQLConf.DEFAULT_CATALOG.key, "testcat_no_namspace") val exception = intercept[AnalysisException] { sql("SHOW NAMESPACES")