[SPARK-57021][CONNECT][PYTHON] Add SQLContext wrapper for Spark Connect

dev/sparktestsupport/modules.py

@@ -572,6 +572,7 @@ def __hash__(self):
         "pyspark.sql.tests.test_column",
         "pyspark.sql.tests.test_conf",
         "pyspark.sql.tests.test_context",
+        "pyspark.sql.tests.test_sql_context",
         "pyspark.sql.tests.test_dataframe",
         "pyspark.sql.tests.test_collection",
         "pyspark.sql.tests.test_creation",
@@ -1165,6 +1166,8 @@ def __hash__(self):
         "pyspark.sql.tests.connect.test_parity_geometrytype",
         "pyspark.sql.tests.connect.test_parity_datasources",
         "pyspark.sql.tests.connect.test_parity_errors",
+        "pyspark.sql.tests.connect.test_connect_context",
+        "pyspark.sql.tests.connect.test_parity_sql_context",
         "pyspark.sql.tests.connect.test_parity_catalog",
         "pyspark.sql.tests.connect.test_parity_conf",
         "pyspark.sql.tests.connect.test_parity_serde",

python/docs/source/reference/pyspark.sql/index.rst

@@ -45,3 +45,4 @@ This page gives an overview of all public Spark SQL API.
     protobuf
     datasource
     stateful_processor
+    legacy

python/docs/source/reference/pyspark.sql/legacy.rst

@@ -0,0 +1,78 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+====================
+Legacy Entry Points
+====================
+.. currentmodule:: pyspark.sql
+
+:class:`SQLContext` was the primary entry point for Spark SQL in Spark 1.x.
+As of Spark 2.0, it has been replaced by :class:`SparkSession`.
+These classes are retained for backward compatibility only.
+
+.. deprecated:: 3.0.0
+    Use :func:`SparkSession.builder.getOrCreate` instead.
+
+.. note::
+    Under Spark Connect, :meth:`SQLContext.registerJavaFunction` and the whole
+    :class:`HiveContext` are not supported and raise
+    :class:`~pyspark.errors.PySparkNotImplementedError`,
+    since they rely on a JVM ``SparkContext`` that does not exist in Connect mode.
+
+SQLContext
+----------
+
+.. autosummary::
+    :toctree: api/
+
+    SQLContext
+
+.. autosummary::
+    :toctree: api/
+
+    SQLContext.getOrCreate
+    SQLContext.newSession
+    SQLContext.setConf
+    SQLContext.getConf
+    SQLContext.udf
+    SQLContext.udtf
+    SQLContext.range
+    SQLContext.registerFunction
+    SQLContext.registerJavaFunction
+    SQLContext.createDataFrame
+    SQLContext.registerDataFrameAsTable
+    SQLContext.dropTempTable
+    SQLContext.createExternalTable
+    SQLContext.sql
+    SQLContext.table
+    SQLContext.tables
+    SQLContext.tableNames
+    SQLContext.cacheTable
+    SQLContext.uncacheTable
+    SQLContext.clearCache
+    SQLContext.read
+    SQLContext.readStream
+    SQLContext.streams
+
+HiveContext
+-----------
+
+.. autosummary::
+    :toctree: api/
+
+    HiveContext

python/pyspark/sql/connect/client/core.py

@@ -2633,3 +2633,32 @@ def clone(self, new_session_id: Optional[str] = None) -> "SparkConnectClient":
         # Ensure the session ID is correctly set from the response
         new_client._session_id = response.new_session_id
         return new_client
+
+    def newSession(self) -> "SparkConnectClient":
+        """
+        Create a new client against the same endpoint with a fresh, independent server-side
+        session that does NOT inherit any state from this client's session. Unlike
+        :meth:`clone`, no state (SQL configurations, temporary views, registered functions,
+        catalog state) is copied over, and no server round-trip is made: the new client is
+        built from a copy of this client's connection configuration with the session ID
+        cleared, so a fresh session ID is generated and the server lazily creates an empty
+        isolated session for it.
+
+        Returns
+        -------
+        SparkConnectClient
+            A new SparkConnectClient instance bound to a fresh, empty session.
+        """
+        # Reuse the same connection configuration (endpoint, channel options, metadata,
+        # user) but drop the session ID so the constructor generates a fresh UUID.
+        new_connection = copy.deepcopy(self._builder)
+        new_connection._params.pop(ChannelBuilder.PARAM_SESSION_ID, None)
+        # Only server-side session state is left behind: client-side behavior such as
+        # registered session hooks and RPC deadlines carries over, as in clone().
+        return SparkConnectClient(
+            connection=new_connection,
+            user_id=self._user_id,
+            use_reattachable_execute=self._use_reattachable_execute,
+            session_hooks=self._session_hooks,
+            rpc_deadlines=self._rpc_deadlines,
+        )