From 98a942c093f01674a25c5fd8d15186406040491b Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Sun, 26 Jul 2026 19:58:24 -0700 Subject: [PATCH 1/8] [GH-3180] Implement distributed GeoPandas spatial subsetting --- docs/tutorial/geopandas-api.md | 20 +- docs/tutorial/geopandas-api.zh.md | 20 +- python/sedona/spark/geopandas/__init__.py | 2 +- python/sedona/spark/geopandas/base.py | 110 ++++ python/sedona/spark/geopandas/geodataframe.py | 6 +- python/sedona/spark/geopandas/geoseries.py | 7 - .../sedona/spark/geopandas/tools/__init__.py | 2 + python/sedona/spark/geopandas/tools/clip.py | 529 ++++++++++++++++++ .../geopandas/test_spatial_subsetting.py | 323 +++++++++++ 9 files changed, 996 insertions(+), 23 deletions(-) create mode 100644 python/sedona/spark/geopandas/tools/clip.py create mode 100644 python/tests/geopandas/test_spatial_subsetting.py diff --git a/docs/tutorial/geopandas-api.md b/docs/tutorial/geopandas-api.md index 1c99c8711d5..d6dc01521d7 100644 --- a/docs/tutorial/geopandas-api.md +++ b/docs/tutorial/geopandas-api.md @@ -181,7 +181,8 @@ nyc_buildings.head() ### Spatial Filtering -Use spatial indexing and filtering methods. Note that `cx` spatial indexing is not yet implemented in the current version: +Use `cx` for distributed coordinate-based filtering and `clip` when the +matching geometries should also be cut to the mask: ```python from shapely.geometry import box @@ -194,14 +195,15 @@ central_park_bbox = box( 40.789, # top-right (longitude, latitude) ) -# Filter buildings within the bounding box using spatial index -# Note: This requires collecting data to driver for spatial filtering -# For large datasets, consider using spatial joins instead -buildings_sample = nyc_buildings.sample(1000) # Sample for demonstration -central_park_buildings = buildings_sample[ - buildings_sample.geometry.intersects(central_park_bbox) +# Select features that intersect the coordinate bounds +central_park_buildings = nyc_buildings.cx[ + -73.973:-73.951, + 40.764:40.789, ] +# Cut the selected geometries to the exact polygon boundary +central_park_buildings = central_park_buildings.clip(central_park_bbox) + # Display results print( central_park_buildings[["BUILD_ID", "PROP_ADDR", "height_val", "geometry"]].head() @@ -307,6 +309,8 @@ The GeoPandas API for Apache Sedona has implemented **39 GeoSeries functions** a ### Spatial Operations - `sjoin()` - Spatial joins with various predicates +- `cx` - Coordinate-based spatial filtering +- `clip()` - Clip geometries with scalar, rectangular, or distributed masks - `buffer()` - Geometric buffering - `distance()` - Distance calculations - `intersects()`, `contains()`, `within()` - Spatial predicates @@ -333,6 +337,8 @@ The GeoPandas API for Apache Sedona has implemented **39 GeoSeries functions** a - `intersects()`, `contains()`, `within()` - Spatial predicates - `intersection()` - Geometric intersection - `make_valid()` - Geometry validation and repair +- `cx` - Coordinate-based spatial filtering +- `clip()` - Distributed geometry clipping - `sindex` - Spatial indexing (limited functionality) ### Data Conversion diff --git a/docs/tutorial/geopandas-api.zh.md b/docs/tutorial/geopandas-api.zh.md index fe6b3568613..82522d04c7e 100644 --- a/docs/tutorial/geopandas-api.zh.md +++ b/docs/tutorial/geopandas-api.zh.md @@ -181,7 +181,8 @@ nyc_buildings.head() ### 空间过滤 -使用空间索引与过滤方法。注意：当前版本尚未实现 `cx` 空间索引： +使用 `cx` 进行分布式坐标范围筛选；如需将相交的几何裁剪到掩膜边界， +可继续使用 `clip`： ```python from shapely.geometry import box @@ -194,14 +195,15 @@ central_park_bbox = box( 40.789, # 右上角（经度，纬度） ) -# 使用空间索引筛选边界框内的建筑 -# 注意：该写法需要把数据收集到 driver 才能进行空间过滤 -# 对于大规模数据，建议改用空间连接（spatial join） -buildings_sample = nyc_buildings.sample(1000) # 演示用：抽样 1000 行 -central_park_buildings = buildings_sample[ - buildings_sample.geometry.intersects(central_park_bbox) +# 筛选与坐标范围相交的要素 +central_park_buildings = nyc_buildings.cx[ + -73.973:-73.951, + 40.764:40.789, ] +# 将筛选后的几何裁剪到精确的多边形边界 +central_park_buildings = central_park_buildings.clip(central_park_bbox) + # 显示结果 print( central_park_buildings[["BUILD_ID", "PROP_ADDR", "height_val", "geometry"]].head() @@ -307,6 +309,8 @@ Apache Sedona 的 GeoPandas API 已实现 **39 个 GeoSeries 函数** 与 **10 ### 空间操作 - `sjoin()` —— 多种谓词的空间连接 +- `cx` —— 基于坐标范围的空间筛选 +- `clip()` —— 使用标量、矩形或分布式掩膜裁剪几何 - `buffer()` —— 几何缓冲 - `distance()` —— 距离计算 - `intersects()`、`contains()`、`within()` —— 空间谓词 @@ -333,6 +337,8 @@ Apache Sedona 的 GeoPandas API 已实现 **39 个 GeoSeries 函数** 与 **10 - `intersects()`、`contains()`、`within()` —— 空间谓词 - `intersection()` —— 几何相交 - `make_valid()` —— 几何校验与修复 +- `cx` —— 基于坐标范围的空间筛选 +- `clip()` —— 分布式几何裁剪 - `sindex` —— 空间索引（功能有限） ### 数据转换 diff --git a/python/sedona/spark/geopandas/__init__.py b/python/sedona/spark/geopandas/__init__.py index cf6c0af175d..a89d732f3f8 100644 --- a/python/sedona/spark/geopandas/__init__.py +++ b/python/sedona/spark/geopandas/__init__.py @@ -23,7 +23,7 @@ from sedona.spark.geopandas.geoseries import GeoSeries from sedona.spark.geopandas.geodataframe import GeoDataFrame -from sedona.spark.geopandas.tools import sjoin +from sedona.spark.geopandas.tools import clip, sjoin from sedona.spark.geopandas.io import read_file, read_parquet diff --git a/python/sedona/spark/geopandas/base.py b/python/sedona/spark/geopandas/base.py index b049b249bea..fa62d08be47 100644 --- a/python/sedona/spark/geopandas/base.py +++ b/python/sedona/spark/geopandas/base.py @@ -78,6 +78,47 @@ def sindex(self) -> "SpatialIndex": """ return _delegate_to_geometry_column("sindex", self) + @property + def cx(self): + """Select geometries that intersect a coordinate bounding box. + + The indexer accepts an x slice followed by a y slice. Slice bounds are + inclusive, may be open-ended, and may be written in either order. + Non-``None`` slice steps are ignored with a warning, matching + GeoPandas. + + Returns + ------- + _CoordinateIndexer + Coordinate indexer returning the same distributed type as this + object. + + Examples + -------- + >>> from sedona.spark.geopandas import GeoSeries + >>> from shapely.geometry import Point + >>> s = GeoSeries([Point(0, 0), Point(1, 1), Point(2, 2)]) + >>> s.cx[0.5:2, 0.5:1.5] + 1 POINT (1 1) + dtype: geometry + + Open-ended slices are supported: + + >>> s.cx[1:, :] + 1 POINT (1 1) + 2 POINT (2 2) + dtype: geometry + + Notes + ----- + Selection is evaluated with native Spark and Sedona expressions. Open + bounds are derived with a one-row distributed bounds aggregation; no + geometry rows are collected to the driver. + """ + from sedona.spark.geopandas.tools.clip import _CoordinateIndexer + + return _CoordinateIndexer(self) + @property def has_sindex(self): """Check the existence of the spatial index without generating it. @@ -3796,6 +3837,75 @@ def clip_by_rect(self, xmin, ymin, xmax, ymax): "clip_by_rect", self, xmin, ymin, xmax, ymax ) + def clip(self, mask, keep_geom_type=False, sort=False): + """Clip geometries to a polygonal or rectangular mask. + + Distributed ``GeoSeries`` and ``GeoDataFrame`` masks are dissolved on + the cluster before clipping. A four-value list-like mask is interpreted + as ``(minx, miny, maxx, maxy)``. Both distributed layers should use the + same Coordinate Reference System (CRS); a mismatch emits a warning. + + Parameters + ---------- + mask : GeoDataFrame, GeoSeries, Polygon, MultiPolygon, or list-like + Polygonal mask, or four rectangle bounds. Distributed masks are + dissolved into one geometry using ``ST_Union_Aggr``. + keep_geom_type : bool, default False + If True, retain only the input geometry family when clipping + creates lower-dimensional geometries. + sort : bool, default False + If True, return matching rows in their original positional order. + + Returns + ------- + GeoDataFrame or GeoSeries + Clipped data of the same type as the caller. Index values, + non-geometry columns, active geometry, CRS, and SRID are preserved. + + Examples + -------- + >>> from sedona.spark.geopandas import GeoDataFrame + >>> from shapely.geometry import LineString, Point, box + >>> gdf = GeoDataFrame( + ... { + ... "name": ["line", "point"], + ... "geometry": [ + ... LineString([(0, 0), (2, 2)]), + ... Point(3, 3), + ... ], + ... } + ... ) + >>> gdf.clip(box(0, 0, 1, 1)) + name geometry + 0 line LINESTRING (0 0, 1 1) + + Rectangle bounds can be supplied directly: + + >>> gdf.geometry.clip((0, 0, 1, 1)) + 0 LINESTRING (0 0, 1 1) + Name: geometry, dtype: geometry + + See Also + -------- + GeoSeries.clip_by_rect + geopandas.clip + + Notes + ----- + Sedona does not expose a separate fast ``ClipByBox`` operation. + Four-value rectangle masks therefore use native ``ST_MakeEnvelope`` + and ``ST_Intersection``. Boundary-only intersections can consequently + differ from GeoPandas' fast, possibly dirty rectangle path. + """ + from sedona.spark.geopandas.tools.clip import clip + + return clip( + self, + mask, + keep_geom_type=keep_geom_type, + sort=sort, + ) + def difference(self, other, align=None): """Returns a ``GeoSeries`` of the points in each aligned geometry that are not in `other`. diff --git a/python/sedona/spark/geopandas/geodataframe.py b/python/sedona/spark/geopandas/geodataframe.py index 68429c93e22..52681ec7489 100644 --- a/python/sedona/spark/geopandas/geodataframe.py +++ b/python/sedona/spark/geopandas/geodataframe.py @@ -396,7 +396,11 @@ def _get_geometry(self) -> sgpd.GeoSeries: ) raise MissingGeometryColumnError(msg) - return self[self._geometry_column_name] + geometry = self[self._geometry_column_name] + empty_crs_source = getattr(self, "_empty_crs_source", None) + if empty_crs_source is not None: + geometry._empty_crs_source = empty_crs_source + return geometry def _set_geometry(self, col): # This check is included in the original geopandas. Note that this prevents assigning a str to the property diff --git a/python/sedona/spark/geopandas/geoseries.py b/python/sedona/spark/geopandas/geoseries.py index 7ced034c51f..fa2230d9b24 100644 --- a/python/sedona/spark/geopandas/geoseries.py +++ b/python/sedona/spark/geopandas/geoseries.py @@ -4033,13 +4033,6 @@ def to_arrow(self, geometry_encoding="WKB", interleaved=True, include_z=None): include_z=include_z, ) - def clip(self, mask, keep_geom_type: bool = False, sort=False) -> "GeoSeries": - raise NotImplementedError( - _not_implemented_error( - "clip", "Clips geometries to the bounds of a mask geometry." - ) - ) - def to_file( self, path: str, diff --git a/python/sedona/spark/geopandas/tools/__init__.py b/python/sedona/spark/geopandas/tools/__init__.py index 05067bba3de..e154327f7da 100644 --- a/python/sedona/spark/geopandas/tools/__init__.py +++ b/python/sedona/spark/geopandas/tools/__init__.py @@ -16,7 +16,9 @@ # under the License. from .sjoin import sjoin +from .clip import clip __all__ = [ + "clip", "sjoin", ] diff --git a/python/sedona/spark/geopandas/tools/clip.py b/python/sedona/spark/geopandas/tools/clip.py new file mode 100644 index 00000000000..653e70171b1 --- /dev/null +++ b/python/sedona/spark/geopandas/tools/clip.py @@ -0,0 +1,529 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Distributed coordinate subsetting and clipping helpers.""" + +from __future__ import annotations + +import numbers +import typing +import warnings + +import geopandas as gpd +import numpy as np +from pandas.api.types import is_list_like +from pyspark.pandas.frame import DataFrame as PandasOnSparkDataFrame +from pyspark.pandas.internal import InternalFrame, NATURAL_ORDER_COLUMN_NAME +from pyspark.pandas.series import first_series +from pyspark.pandas.utils import scol_for, verify_temp_column_name +from pyspark.sql import functions as F +from shapely.geometry import MultiPolygon, Polygon +from shapely.geometry.base import BaseGeometry + +from sedona.spark.sql import st_aggregates as sta +from sedona.spark.sql import st_constructors as stc +from sedona.spark.sql import st_functions as stf +from sedona.spark.sql import st_predicates as stp + +_POINT_TYPES = ("ST_Point", "ST_MultiPoint") +_LINE_TYPES = ("ST_LineString", "ST_MultiLineString") +_POLYGON_TYPES = ("ST_Polygon", "ST_MultiPolygon") +_GEOMETRY_COLLECTION = "ST_GeometryCollection" + + +def _temporary_column_name(sdf, base: str, reserved: set[str]) -> str: + """Return a Spark column name that cannot collide with user columns.""" + suffix = 0 + candidate = f"__geopandas_{base}__" + while candidate in reserved: + suffix += 1 + candidate = f"__geopandas_{base}_{suffix}__" + reserved.add(candidate) + return typing.cast(str, verify_temp_column_name(sdf, candidate)) + + +def _geometry_context(obj): + """Resolve an object's Spark frame and active physical geometry column.""" + internal = obj._internal.resolved_copy + geometry = obj.geometry + geometry_internal = geometry._internal.resolved_copy + geometry_name = geometry_internal.data_spark_column_names[0] + return internal, internal.spark_frame, geometry_name + + +def _rebuild_like(obj, internal, sdf): + """Rebuild a distributed GeoSeries/GeoDataFrame from a projected frame.""" + from sedona.spark.geopandas.geodataframe import GeoDataFrame + from sedona.spark.geopandas.geoseries import GeoSeries + + result_internal = internal.copy( + spark_frame=sdf, + index_spark_columns=[ + scol_for(sdf, name) for name in internal.index_spark_column_names + ], + data_spark_columns=[ + scol_for(sdf, name) for name in internal.data_spark_column_names + ], + ) + + if isinstance(obj, GeoSeries): + result = GeoSeries(first_series(PandasOnSparkDataFrame(result_internal))) + result._empty_crs_source = obj + return result + + result = GeoDataFrame(PandasOnSparkDataFrame(result_internal)) + result._geometry_column_name = obj.active_geometry_name + object.__setattr__(result, "_empty_crs_source", obj.geometry) + return result + + +def _project_replaced_geometry(sdf, geometry_name: str, geometry): + """Select the original columns, replacing only the active geometry.""" + return sdf.select( + *[ + (geometry.alias(name) if name == geometry_name else scol_for(sdf, name)) + for name in sdf.columns + ] + ) + + +def _sort_by_natural_order(sdf): + """Restore input positional order, matching GeoPandas' sorted index query.""" + ordered = sdf.orderBy(scol_for(sdf, NATURAL_ORDER_COLUMN_NAME).asc()).drop( + NATURAL_ORDER_COLUMN_NAME + ) + ordered = InternalFrame.attach_distributed_sequence_column( + ordered, NATURAL_ORDER_COLUMN_NAME + ) + return ordered.select(*[scol_for(ordered, name) for name in sdf.columns]) + + +def _normalize_slice_bound(value, axis: str): + if value is None: + return None + if isinstance(value, (str, bytes, bytearray)) or not isinstance( + value, numbers.Real + ): + raise TypeError(f"{axis} slice bounds must be numeric or None") + return float(value) + + +class _CoordinateIndexer: + """Coordinate-based indexer shared by GeoSeries and GeoDataFrame.""" + + def __init__(self, obj): + self.obj = obj + + def __getitem__(self, key): + if not isinstance(key, tuple) or len(key) != 2: + raise TypeError("Coordinate based indexing requires an x and y slice") + + xs, ys = key + if type(xs) is not slice: + xs = slice(xs, xs) + if type(ys) is not slice: + ys = slice(ys, ys) + + if xs.step is not None or ys.step is not None: + warnings.warn( + "Ignoring step - full interval is used.", + UserWarning, + stacklevel=2, + ) + + xmin = _normalize_slice_bound(xs.start, "x") + xmax = _normalize_slice_bound(xs.stop, "x") + ymin = _normalize_slice_bound(ys.start, "y") + ymax = _normalize_slice_bound(ys.stop, "y") + + # Shapely's box constructor, which GeoPandas uses, normalizes reversed + # slice bounds. + if xmin is not None and xmax is not None and xmin > xmax: + xmin, xmax = xmax, xmin + if ymin is not None and ymax is not None and ymin > ymax: + ymin, ymax = ymax, ymin + + return _coordinate_subset(self.obj, xmin, ymin, xmax, ymax) + + +def _coordinate_subset(obj, xmin, ymin, xmax, ymax): + """Filter rows by exact intersection with a distributed bounding box.""" + internal, source_sdf, geometry_name = _geometry_context(obj) + reserved = set(source_sdf.columns) + source_geometry = scol_for(source_sdf, geometry_name) + bounded_sdf = source_sdf + + bound_values = { + "xmin": xmin, + "ymin": ymin, + "xmax": xmax, + "ymax": ymax, + } + missing_bounds = [name for name, value in bound_values.items() if value is None] + + if missing_bounds: + aggregate_names = { + name: _temporary_column_name(source_sdf, f"cx_{name}", reserved) + for name in missing_bounds + } + valid_geometry = source_geometry.isNotNull() & ~stf.ST_IsEmpty(source_geometry) + aggregate_builders = { + "xmin": lambda: F.min(F.when(valid_geometry, stf.ST_XMin(source_geometry))), + "ymin": lambda: F.min(F.when(valid_geometry, stf.ST_YMin(source_geometry))), + "xmax": lambda: F.max(F.when(valid_geometry, stf.ST_XMax(source_geometry))), + "ymax": lambda: F.max(F.when(valid_geometry, stf.ST_YMax(source_geometry))), + } + bounds_sdf = source_sdf.agg( + *[ + aggregate_builders[name]().alias(aggregate_names[name]) + for name in missing_bounds + ] + ) + bounded_sdf = source_sdf.crossJoin(F.broadcast(bounds_sdf)) + source_geometry = scol_for(bounded_sdf, geometry_name) + for name in missing_bounds: + bound_values[name] = scol_for(bounded_sdf, aggregate_names[name]) + + envelope = stc.ST_MakeEnvelope( + bound_values["xmin"], + bound_values["ymin"], + bound_values["xmax"], + bound_values["ymax"], + stf.ST_SRID(source_geometry), + ) + filtered = bounded_sdf.where(stp.ST_Intersects(source_geometry, envelope)).select( + *[scol_for(bounded_sdf, name) for name in source_sdf.columns] + ) + return _rebuild_like(obj, internal, filtered) + + +def _normalize_rectangle(mask): + values = tuple(mask) + if len(values) != 4: + raise TypeError( + "If 'mask' is list-like, it must have four values " + "(minx, miny, maxx, maxy)" + ) + + normalized = [] + for value in values: + if isinstance(value, (str, bytes, bytearray)) or not isinstance( + value, numbers.Real + ): + raise TypeError("Rectangle mask values must be numeric") + normalized.append(float(value)) + return tuple(normalized) + + +def _mask_is_list_like_rectangle(mask) -> bool: + from sedona.spark.geopandas.geodataframe import GeoDataFrame + from sedona.spark.geopandas.geoseries import GeoSeries + + return is_list_like(mask) and not isinstance( + mask, + ( + GeoDataFrame, + GeoSeries, + gpd.GeoDataFrame, + gpd.GeoSeries, + Polygon, + MultiPolygon, + ), + ) + + +def _as_distributed_mask(mask): + """Convert local GeoPandas masks without collecting distributed masks.""" + from sedona.spark.geopandas.geodataframe import GeoDataFrame + from sedona.spark.geopandas.geoseries import GeoSeries + + if isinstance(mask, gpd.GeoDataFrame): + return GeoSeries(mask.geometry, crs=mask.crs) + if isinstance(mask, gpd.GeoSeries): + return GeoSeries(mask, crs=mask.crs) + if isinstance(mask, (GeoDataFrame, GeoSeries)): + return mask + return None + + +def _warn_crs_mismatch(obj, mask): + if obj.crs == mask.crs: + return + + try: + from geopandas.array import _crs_mismatch_warn + except ImportError: + warnings.warn( + f"CRS mismatch between the CRS of left geometries ({obj.crs}) " + f"and right geometries ({mask.crs}).", + UserWarning, + stacklevel=3, + ) + else: + _crs_mismatch_warn(obj, mask, stacklevel=3) + + +def _mask_expression(source_sdf, mask, reserved): + """Return a source frame and a mask column expression.""" + rectangle = _mask_is_list_like_rectangle(mask) + if rectangle: + values = _normalize_rectangle(mask) + if ( + any(np.isnan(value) for value in values) + or values[0] >= values[2] + or values[1] >= values[3] + ): + return ( + source_sdf, + stc.ST_GeomFromWKT(F.lit("POLYGON EMPTY")), + True, + ) + return source_sdf, stc.ST_MakeEnvelope(*values), True + + distributed_mask = _as_distributed_mask(mask) + if distributed_mask is not None: + _, mask_sdf, mask_geometry_name = _geometry_context(distributed_mask) + mask_name = _temporary_column_name(source_sdf, "clip_mask", reserved) + dissolved_mask = mask_sdf.agg( + sta.ST_Union_Aggr(scol_for(mask_sdf, mask_geometry_name)).alias(mask_name) + ) + joined = source_sdf.crossJoin(F.broadcast(dissolved_mask)) + return joined, scol_for(joined, mask_name), False + + if isinstance(mask, (Polygon, MultiPolygon)): + return ( + source_sdf, + stc.ST_GeomFromWKT(F.lit(mask.wkt)), + False, + ) + + if isinstance(mask, BaseGeometry): + raise TypeError( + "'mask' should be a GeoDataFrame, GeoSeries, " "(Multi)Polygon or list-like" + ) + raise TypeError( + "'mask' should be a GeoDataFrame, GeoSeries, " + f"(Multi)Polygon or list-like, got {type(mask)}" + ) + + +def _geometry_family(geometry): + geometry_type = stf.ST_GeometryType(geometry) + return ( + F.when(geometry_type.isin(*_POINT_TYPES), F.lit("point")) + .when(geometry_type.isin(*_LINE_TYPES), F.lit("line")) + .when(geometry_type.isin(*_POLYGON_TYPES), F.lit("polygon")) + ) + + +def _source_geometry_family(source_sdf, geometry_name): + """Summarize source geometry families without collecting geometries.""" + geometry = scol_for(source_sdf, geometry_name) + geometry_type = stf.ST_GeometryType(geometry) + family = _geometry_family(geometry) + summary = source_sdf.agg( + F.max( + F.when(geometry_type == _GEOMETRY_COLLECTION, F.lit(1)).otherwise(F.lit(0)) + ).alias("has_collection"), + F.countDistinct(family).alias("family_count"), + F.first(family, ignorenulls=True).alias("family"), + ).first() + return summary.has_collection, summary.family_count, summary.family + + +def _clipped_geometry_summary(sdf, geometry_name): + """Return collection presence and basic family count for clipped rows.""" + geometry = scol_for(sdf, geometry_name) + geometry_type = stf.ST_GeometryType(geometry) + family = _geometry_family(geometry) + summary = sdf.agg( + F.max( + F.when(geometry_type == _GEOMETRY_COLLECTION, F.lit(1)).otherwise(F.lit(0)) + ).alias("has_collection"), + F.countDistinct(family).alias("family_count"), + ).first() + return summary.has_collection, summary.family_count + + +def _keep_only_family(internal, sdf, geometry_name, family): + """Explode new collections and retain only the source geometry family.""" + reserved = set(sdf.columns) + parent_order_name = _temporary_column_name(sdf, "clip_parent_order", reserved) + position_name = _temporary_column_name(sdf, "clip_position", reserved) + geometry_value_name = _temporary_column_name(sdf, "clip_geometry", reserved) + geometry = scol_for(sdf, geometry_name) + geometry_parts = F.when( + stf.ST_GeometryType(geometry) == _GEOMETRY_COLLECTION, + stf.ST_Dump(geometry), + ).otherwise(F.array(geometry)) + + expanded = sdf.select( + *[ + scol_for(sdf, name) + for name in sdf.columns + if name not in (geometry_name, NATURAL_ORDER_COLUMN_NAME) + ], + scol_for(sdf, NATURAL_ORDER_COLUMN_NAME).alias(parent_order_name), + F.posexplode(geometry_parts).alias(position_name, geometry_value_name), + ) + + allowed_types = { + "point": _POINT_TYPES, + "line": _LINE_TYPES, + "polygon": _POLYGON_TYPES, + }[family] + expanded = ( + expanded.where( + stf.ST_GeometryType(scol_for(expanded, geometry_value_name)).isin( + *allowed_types + ) + ) + .select( + *[ + ( + scol_for(expanded, geometry_value_name).alias(geometry_name) + if name == geometry_name + else scol_for(expanded, name) + ) + for name in internal.spark_frame.columns + if name != NATURAL_ORDER_COLUMN_NAME + ], + scol_for(expanded, parent_order_name), + scol_for(expanded, position_name), + ) + .orderBy(parent_order_name, position_name) + .drop(parent_order_name, position_name) + ) + expanded = InternalFrame.attach_distributed_sequence_column( + expanded, NATURAL_ORDER_COLUMN_NAME + ) + return expanded.select( + *[scol_for(expanded, name) for name in internal.spark_frame.columns] + ) + + +def clip(gdf, mask, keep_geom_type: bool = False, sort: bool = False): + """Clip a distributed GeoSeries or GeoDataFrame to a mask. + + The operation uses native Spark and Sedona expressions. Distributed masks + are dissolved on the cluster and are never materialized as geometry rows + in Python. + + Parameters + ---------- + gdf : GeoDataFrame or GeoSeries + Distributed vector data to clip. + mask : GeoDataFrame, GeoSeries, Polygon, MultiPolygon, or list-like + Polygonal mask. A four-value list-like mask is interpreted as + ``(minx, miny, maxx, maxy)``. + keep_geom_type : bool, default False + Retain only the input geometry family when clipping creates lower + dimensional geometries. + sort : bool, default False + Return matching rows in their original positional order. + + Returns + ------- + GeoDataFrame or GeoSeries + Clipped distributed data of the same type as ``gdf``. + + See Also + -------- + GeoDataFrame.clip + GeoSeries.clip + """ + from sedona.spark.geopandas.geodataframe import GeoDataFrame + from sedona.spark.geopandas.geoseries import GeoSeries + + if not isinstance(gdf, (GeoDataFrame, GeoSeries)): + raise TypeError(f"'gdf' should be GeoDataFrame or GeoSeries, got {type(gdf)}") + if not isinstance(keep_geom_type, (bool, np.bool_)): + raise TypeError("'keep_geom_type' must be a boolean") + if not isinstance(sort, (bool, np.bool_)): + raise TypeError("'sort' must be a boolean") + + distributed_mask = _as_distributed_mask(mask) + if distributed_mask is not None: + _warn_crs_mismatch(gdf, distributed_mask) + mask = distributed_mask + + internal, source_sdf, geometry_name = _geometry_context(gdf) + reserved = set(source_sdf.columns) + + source_family = None + if keep_geom_type: + has_collection, family_count, source_family = _source_geometry_family( + source_sdf, geometry_name + ) + if has_collection: + warnings.warn( + "keep_geom_type can not be called on a " + "GeoDataFrame with GeometryCollection.", + UserWarning, + stacklevel=2, + ) + source_family = None + elif family_count > 1: + warnings.warn( + "keep_geom_type can not be called on a mixed type GeoDataFrame.", + UserWarning, + stacklevel=2, + ) + source_family = None + + working_sdf, mask_geometry, rectangle = _mask_expression(source_sdf, mask, reserved) + source_geometry = scol_for(working_sdf, geometry_name) + geometry_type = stf.ST_GeometryType(source_geometry) + clipped_geometry = F.when(geometry_type == "ST_Point", source_geometry).otherwise( + stf.ST_SetSRID( + stf.ST_Intersection(source_geometry, mask_geometry), + stf.ST_SRID(source_geometry), + ) + ) + + clipped_sdf = working_sdf.where(stp.ST_Intersects(source_geometry, mask_geometry)) + clipped_sdf = _project_replaced_geometry( + clipped_sdf, geometry_name, clipped_geometry + ) + clipped_sdf = clipped_sdf.select( + *[scol_for(clipped_sdf, name) for name in source_sdf.columns] + ) + + if rectangle: + clipped_sdf = clipped_sdf.where( + ~stf.ST_IsEmpty(scol_for(clipped_sdf, geometry_name)) + ) + + if source_family is not None: + has_new_collection, clipped_family_count = _clipped_geometry_summary( + clipped_sdf, geometry_name + ) + # GeoPandas filters only when clipping introduces a collection or an + # additional basic family. If every line collapses to a point, for + # example, those points remain even with keep_geom_type=True. + if has_new_collection or clipped_family_count > 1: + clipped_sdf = _keep_only_family( + internal, clipped_sdf, geometry_name, source_family + ) + + if sort: + clipped_sdf = _sort_by_natural_order(clipped_sdf) + + return _rebuild_like(gdf, internal, clipped_sdf) + + +__all__ = ["clip"] diff --git a/python/tests/geopandas/test_spatial_subsetting.py b/python/tests/geopandas/test_spatial_subsetting.py new file mode 100644 index 00000000000..4c88b03eb87 --- /dev/null +++ b/python/tests/geopandas/test_spatial_subsetting.py @@ -0,0 +1,323 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import warnings + +import geopandas as gpd +import numpy as np +import pandas as pd +import pytest +import pyspark.pandas as ps +from shapely.geometry import ( + GeometryCollection, + LineString, + MultiPoint, + Point, + Polygon, + box, +) + +import sedona.spark.geopandas as sgpd +from sedona.spark.geopandas import GeoDataFrame, GeoSeries +from sedona.spark.sql import st_functions as stf +from tests.geopandas.test_geopandas_base import TestGeopandasBase + + +class TestDistributedSpatialSubsetting(TestGeopandasBase): + @staticmethod + def _assert_srid(series, expected): + internal = series._internal.resolved_copy + geometry = internal.data_spark_columns[0] + srids = { + row.srid + for row in internal.spark_frame.select( + stf.ST_SRID(geometry).alias("srid") + ).collect() + if row.srid is not None + } + assert srids == {expected} + + def test_cx_series_matches_geopandas_and_uses_exact_intersection(self): + geometries = [ + Point(0, 0), + LineString([(-2, 2), (2, -2)]), + # Its envelope intersects the query rectangle, but the geometry + # itself does not. + MultiPoint([(-2, -2), (2, 2)]), + Point(), + None, + ] + index = pd.MultiIndex.from_tuples( + [("a", 2), ("a", 1), ("b", 2), ("b", 1), ("c", 1)], + names=["group", "position"], + ) + expected_source = gpd.GeoSeries(geometries, index=index, name="shape", crs=4326) + source = GeoSeries(expected_source, crs=4326) + + result = source.cx[-0.5:0.5, -0.5:0.5] + expected = expected_source.cx[-0.5:0.5, -0.5:0.5] + + self.check_sgpd_equals_gpd(result, expected) + assert list(result.to_geopandas().index) == list(expected.index) + assert result.crs == expected.crs + self._assert_srid(result, 4326) + + def test_cx_open_reversed_numeric_and_step_slices(self): + geometries = [ + Point(0, 0), + Point(1, 1), + Point(2, 2), + Point(), + None, + ] + expected_source = gpd.GeoSeries(geometries, name="shape", crs=3857) + source = GeoSeries(expected_source, crs=3857) + + for key in [ + (slice(1, None), slice(None)), + (slice(2, 1), slice(None)), + (1, 1), + (slice(None), slice(None)), + ]: + self.check_sgpd_equals_gpd(source.cx[key], expected_source.cx[key]) + + with pytest.warns(UserWarning, match="Ignoring step"): + result = source.cx[0:2:10, 0:2] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + expected = expected_source.cx[0:2:10, 0:2] + self.check_sgpd_equals_gpd(result, expected) + + with pytest.raises(TypeError): + source.cx[0:1] + with pytest.raises(TypeError, match="numeric"): + source.cx["left":"right", :] + + def test_cx_geodataframe_preserves_columns_active_geometry_and_crs(self): + index = pd.MultiIndex.from_tuples( + [("b", 2), ("a", 2), ("a", 1)], names=["group", "position"] + ) + expected_source = gpd.GeoDataFrame( + { + "value": [10, 20, 30], + "backup": [Point(10, 10), Point(11, 11), Point(12, 12)], + "shape": [Point(2, 2), Point(0, 0), Point(1, 1)], + }, + geometry="shape", + index=index, + crs=4326, + ) + with ps.option_context("compute.ops_on_diff_frames", True): + source = GeoDataFrame(expected_source) + + result = source.cx[:1, :1] + expected = expected_source.cx[:1, :1] + + self.check_sgpd_df_equals_gpd_df(result, expected) + assert result.active_geometry_name == "shape" + assert result.crs == expected.crs + self._assert_srid(result.geometry, 4326) + + @pytest.mark.parametrize("use_top_level", [False, True]) + def test_clip_series_scalar_polygon_matches_geopandas(self, use_top_level): + geometries = [ + Point(0, 0), + LineString([(-1, 0.5), (2, 0.5)]), + Polygon([(0.5, -1), (2, -1), (2, 2), (0.5, 2)]), + Point(3, 3), + Point(), + None, + ] + index = pd.Index([6, 5, 4, 3, 2, 1], name="feature_id") + expected_source = gpd.GeoSeries(geometries, index=index, name="shape", crs=4326) + source = GeoSeries(expected_source, crs=4326) + mask = box(0, 0, 1, 1) + + if use_top_level: + result = sgpd.clip(source, mask) + else: + result = source.clip(mask) + expected = gpd.clip(expected_source, mask) + + self.check_sgpd_equals_gpd(result, expected) + assert result.crs == expected.crs + self._assert_srid(result, 4326) + + def test_clip_distributed_mask_is_dissolved_on_cluster(self): + expected_source = gpd.GeoSeries( + [ + LineString([(-1, 0.5), (2, 0.5)]), + Point(0.25, 0.25), + Point(2, 2), + ], + index=pd.Index([3, 2, 1], name="feature_id"), + name="shape", + crs=3857, + ) + expected_mask = gpd.GeoSeries( + [box(0, 0, 0.5, 1), box(0.5, 0, 1, 1)], + name="mask", + crs=3857, + ) + source = GeoSeries(expected_source, crs=3857) + mask = GeoSeries(expected_mask, crs=3857) + + result = source.clip(mask) + expected = gpd.clip(expected_source, expected_mask) + + self.check_sgpd_equals_gpd(result, expected) + plan = result._internal.spark_frame._jdf.queryExecution().toString() + assert "union_aggr" in plan.lower() + assert "PythonUDF" not in plan + + def test_clip_rectangle_and_empty_results_preserve_crs(self): + expected_source = gpd.GeoSeries( + [ + LineString([(-1, 0.5), (2, 0.5)]), + Point(0.25, 0.25), + Point(2, 2), + ], + name="shape", + crs=4326, + ) + source = GeoSeries(expected_source, crs=4326) + rectangle = (0, 0, 1, 1) + + self.check_sgpd_equals_gpd( + source.clip(rectangle), + gpd.clip(expected_source, rectangle), + ) + + for invalid_rectangle in [ + (1, 0, 0, 1), + (0, 1, 1, 0), + (0, 0, 0, 1), + (0, 0, 1, 0), + (np.nan, 0, np.nan, 1), + ]: + invalid_result = source.clip(invalid_rectangle) + assert len(invalid_result) == 0 + assert invalid_result.crs == expected_source.crs + + empty_series = source.clip(box(10, 10, 11, 11)) + assert len(empty_series) == 0 + assert empty_series.crs == expected_source.crs + + source_frame = source.to_geoframe("geometry") + empty_frame = source_frame.clip(box(10, 10, 11, 11)) + assert len(empty_frame) == 0 + assert empty_frame.crs == expected_source.crs + assert empty_frame.active_geometry_name == "geometry" + + def test_clip_geodataframe_preserves_index_columns_and_active_geometry(self): + index = pd.MultiIndex.from_tuples( + [("b", 2), ("a", 2), ("a", 1), ("c", 1)], + names=["group", "position"], + ) + expected_source = gpd.GeoDataFrame( + { + "value": [10, 20, 30, 40], + "label": ["ten", "twenty", "thirty", "forty"], + "shape": [ + Point(2, 2), + LineString([(-1, 0.5), (2, 0.5)]), + Point(0.5, 0.5), + Point(3, 3), + ], + }, + geometry="shape", + index=index, + crs=4326, + ) + expected_mask = gpd.GeoDataFrame( + {"mask_id": [1, 2]}, + geometry=[box(0, 0, 0.5, 1), box(0.5, 0, 1, 1)], + crs=4326, + ) + with ps.option_context("compute.ops_on_diff_frames", True): + source = GeoDataFrame(expected_source) + mask = GeoDataFrame(expected_mask) + + result = source.clip(mask, sort=True) + expected = expected_source.clip(expected_mask, sort=True) + + self.check_sgpd_df_equals_gpd_df(result, expected) + assert result.active_geometry_name == "shape" + assert list(result.to_geopandas().index) == list(expected.index) + assert result.crs == expected.crs + self._assert_srid(result.geometry, 4326) + + def test_clip_keep_geom_type_and_mixed_type_warnings(self): + mask = box(0, 0, 1, 1) + expected_lines = gpd.GeoSeries( + [ + LineString([(1, 0.5), (2, 0.5)]), + LineString([(-1, 0.25), (2, 0.25)]), + ], + index=pd.Index([5, 3], name="feature_id"), + crs=4326, + ) + lines = GeoSeries(expected_lines, crs=4326) + + self.check_sgpd_equals_gpd( + lines.clip(mask, keep_geom_type=True), + expected_lines.clip(mask, keep_geom_type=True), + ) + + collapsed_line = gpd.GeoSeries([LineString([(1, 0.5), (2, 0.5)])], crs=4326) + self.check_sgpd_equals_gpd( + GeoSeries(collapsed_line, crs=4326).clip(mask, keep_geom_type=True), + collapsed_line.clip(mask, keep_geom_type=True), + ) + + expected_mixed = gpd.GeoSeries( + [Point(0.5, 0.5), LineString([(-1, 0.5), (2, 0.5)])], + crs=4326, + ) + mixed = GeoSeries(expected_mixed, crs=4326) + with pytest.warns(UserWarning, match="mixed type"): + result = mixed.clip(mask, keep_geom_type=True) + with pytest.warns(UserWarning, match="mixed type"): + expected = expected_mixed.clip(mask, keep_geom_type=True) + self.check_sgpd_equals_gpd(result, expected) + + expected_collection = gpd.GeoSeries( + [GeometryCollection([Point(0.5, 0.5)])], crs=4326 + ) + collection = GeoSeries(expected_collection, crs=4326) + with pytest.warns(UserWarning, match="GeometryCollection"): + collection.clip(mask, keep_geom_type=True) + + def test_clip_crs_mismatch_warns_and_invalid_inputs_fail(self): + source = GeoSeries([Point(0.5, 0.5)], crs=4326) + mask = GeoSeries([box(0, 0, 1, 1)], crs=3857) + + with pytest.warns(UserWarning, match="CRS mismatch between the CRS"): + source.clip(mask) + + with pytest.raises(TypeError, match="'gdf'"): + sgpd.clip([Point(0, 0)], box(0, 0, 1, 1)) + with pytest.raises(TypeError, match="four values"): + source.clip((0, 0, 1)) + with pytest.raises(TypeError, match="Rectangle mask values"): + source.clip((0, 0, "right", 1)) + with pytest.raises(TypeError, match="'mask'"): + source.clip(Point(0, 0)) + with pytest.raises(TypeError, match="keep_geom_type"): + source.clip(box(0, 0, 1, 1), keep_geom_type="yes") + with pytest.raises(TypeError, match="'sort'"): + source.clip(box(0, 0, 1, 1), sort="yes") From 7de093ba8454d18c7ba4e7a9f92f30710cd179d7 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Sun, 26 Jul 2026 20:21:51 -0700 Subject: [PATCH 2/8] [GH-3180] Fix clipping compatibility and ordering --- python/sedona/spark/geopandas/tools/clip.py | 55 +++++++++---------- .../geopandas/test_spatial_subsetting.py | 31 ++++++++++- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/python/sedona/spark/geopandas/tools/clip.py b/python/sedona/spark/geopandas/tools/clip.py index 653e70171b1..275b1c6ec8b 100644 --- a/python/sedona/spark/geopandas/tools/clip.py +++ b/python/sedona/spark/geopandas/tools/clip.py @@ -27,7 +27,7 @@ import numpy as np from pandas.api.types import is_list_like from pyspark.pandas.frame import DataFrame as PandasOnSparkDataFrame -from pyspark.pandas.internal import InternalFrame, NATURAL_ORDER_COLUMN_NAME +from pyspark.pandas.internal import NATURAL_ORDER_COLUMN_NAME from pyspark.pandas.series import first_series from pyspark.pandas.utils import scol_for, verify_temp_column_name from pyspark.sql import functions as F @@ -103,13 +103,9 @@ def _project_replaced_geometry(sdf, geometry_name: str, geometry): def _sort_by_natural_order(sdf): """Restore input positional order, matching GeoPandas' sorted index query.""" - ordered = sdf.orderBy(scol_for(sdf, NATURAL_ORDER_COLUMN_NAME).asc()).drop( - NATURAL_ORDER_COLUMN_NAME + return sdf.orderBy(scol_for(sdf, NATURAL_ORDER_COLUMN_NAME).asc()).select( + *[scol_for(sdf, name) for name in sdf.columns] ) - ordered = InternalFrame.attach_distributed_sequence_column( - ordered, NATURAL_ORDER_COLUMN_NAME - ) - return ordered.select(*[scol_for(ordered, name) for name in sdf.columns]) def _normalize_slice_bound(value, axis: str): @@ -340,7 +336,9 @@ def _source_geometry_family(source_sdf, geometry_name): F.when(geometry_type == _GEOMETRY_COLLECTION, F.lit(1)).otherwise(F.lit(0)) ).alias("has_collection"), F.countDistinct(family).alias("family_count"), - F.first(family, ignorenulls=True).alias("family"), + F.min_by(family, scol_for(source_sdf, NATURAL_ORDER_COLUMN_NAME)).alias( + "family" + ), ).first() return summary.has_collection, summary.family_count, summary.family @@ -386,30 +384,27 @@ def _keep_only_family(internal, sdf, geometry_name, family): "line": _LINE_TYPES, "polygon": _POLYGON_TYPES, }[family] - expanded = ( - expanded.where( - stf.ST_GeometryType(scol_for(expanded, geometry_value_name)).isin( - *allowed_types - ) - ) - .select( - *[ - ( - scol_for(expanded, geometry_value_name).alias(geometry_name) - if name == geometry_name - else scol_for(expanded, name) - ) - for name in internal.spark_frame.columns - if name != NATURAL_ORDER_COLUMN_NAME - ], - scol_for(expanded, parent_order_name), - scol_for(expanded, position_name), + expanded = expanded.where( + stf.ST_GeometryType(scol_for(expanded, geometry_value_name)).isin( + *allowed_types ) - .orderBy(parent_order_name, position_name) - .drop(parent_order_name, position_name) + ).select( + *[ + ( + scol_for(expanded, geometry_value_name).alias(geometry_name) + if name == geometry_name + else scol_for(expanded, name) + ) + for name in internal.spark_frame.columns + if name != NATURAL_ORDER_COLUMN_NAME + ], + scol_for(expanded, parent_order_name), + scol_for(expanded, position_name), ) - expanded = InternalFrame.attach_distributed_sequence_column( - expanded, NATURAL_ORDER_COLUMN_NAME + expanded = ( + expanded.orderBy(parent_order_name, position_name) + .withColumn(NATURAL_ORDER_COLUMN_NAME, F.monotonically_increasing_id()) + .drop(parent_order_name, position_name) ) return expanded.select( *[scol_for(expanded, name) for name in internal.spark_frame.columns] diff --git a/python/tests/geopandas/test_spatial_subsetting.py b/python/tests/geopandas/test_spatial_subsetting.py index 4c88b03eb87..6da9b800a5d 100644 --- a/python/tests/geopandas/test_spatial_subsetting.py +++ b/python/tests/geopandas/test_spatial_subsetting.py @@ -22,6 +22,7 @@ import pandas as pd import pytest import pyspark.pandas as ps +from packaging.version import parse as parse_version from shapely.geometry import ( GeometryCollection, LineString, @@ -253,7 +254,13 @@ def test_clip_geodataframe_preserves_index_columns_and_active_geometry(self): mask = GeoDataFrame(expected_mask) result = source.clip(mask, sort=True) - expected = expected_source.clip(expected_mask, sort=True) + if parse_version(gpd.__version__) >= parse_version("1.0.0"): + expected = expected_source.clip(expected_mask, sort=True) + else: + expected = expected_source.clip(expected_mask) + expected = expected.loc[ + expected_source.index[expected_source.index.isin(expected.index)] + ] self.check_sgpd_df_equals_gpd_df(result, expected) assert result.active_geometry_name == "shape" @@ -302,6 +309,28 @@ def test_clip_keep_geom_type_and_mixed_type_warnings(self): with pytest.warns(UserWarning, match="GeometryCollection"): collection.clip(mask, keep_geom_type=True) + def test_clip_keep_geom_type_uses_first_geometry_type_including_null(self): + mask = box(0, 0, 1, 1) + expected_source = gpd.GeoSeries( + [ + None, + LineString([(-1, 0.25), (2, 0.25)]), + LineString([(1, 0.5), (2, 0.5)]), + ], + index=pd.Index(["null", "line", "point"], name="feature_id"), + crs=4326, + ) + source_seed = expected_source.copy() + source_seed.iloc[0] = LineString([(0, 0), (1, 1)]) + source = GeoSeries(source_seed, crs=4326) + source.iloc[0] = None + + result = source.clip(mask, keep_geom_type=True) + expected = expected_source.clip(mask, keep_geom_type=True) + + self.check_sgpd_equals_gpd(result, expected) + assert set(result.geom_type.to_pandas()) == {"LineString", "Point"} + def test_clip_crs_mismatch_warns_and_invalid_inputs_fail(self): source = GeoSeries([Point(0.5, 0.5)], crs=4326) mask = GeoSeries([box(0, 0, 1, 1)], crs=3857) From a55f8e21391f11a3f89c60df67fb54458552d7a1 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Sun, 26 Jul 2026 21:23:47 -0700 Subject: [PATCH 3/8] [GH-3180] Remove brittle GeoPandas API counters --- docs/tutorial/geopandas-api.md | 2 +- docs/tutorial/geopandas-api.zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorial/geopandas-api.md b/docs/tutorial/geopandas-api.md index d6dc01521d7..03a3aa6db66 100644 --- a/docs/tutorial/geopandas-api.md +++ b/docs/tutorial/geopandas-api.md @@ -298,7 +298,7 @@ buildings_projected["perimeter"] = buildings_projected.geometry.length ## Supported Operations -The GeoPandas API for Apache Sedona has implemented **39 GeoSeries functions** and **10 GeoDataFrame functions**, covering the most commonly used GeoPandas operations: +The GeoPandas API for Apache Sedona implements the most commonly used GeoSeries and GeoDataFrame operations: ### Data I/O diff --git a/docs/tutorial/geopandas-api.zh.md b/docs/tutorial/geopandas-api.zh.md index 82522d04c7e..a85e5f7570b 100644 --- a/docs/tutorial/geopandas-api.zh.md +++ b/docs/tutorial/geopandas-api.zh.md @@ -298,7 +298,7 @@ buildings_projected["perimeter"] = buildings_projected.geometry.length ## 已支持的操作 -Apache Sedona 的 GeoPandas API 已实现 **39 个 GeoSeries 函数** 与 **10 个 GeoDataFrame 函数**，覆盖了 GeoPandas 中最常用的操作： +Apache Sedona 的 GeoPandas API 已实现最常用的 GeoSeries 与 GeoDataFrame 操作： ### 数据 I/O From 5251628632d8da01e745005a03c7e6613ea0d88f Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Sun, 26 Jul 2026 22:14:28 -0700 Subject: [PATCH 4/8] [GH-3180] Preserve CRS for all-null spatial subsets --- python/sedona/spark/geopandas/geoseries.py | 17 ++++++++++++- .../geopandas/test_spatial_subsetting.py | 24 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/python/sedona/spark/geopandas/geoseries.py b/python/sedona/spark/geopandas/geoseries.py index fa2230d9b24..981c13529b7 100644 --- a/python/sedona/spark/geopandas/geoseries.py +++ b/python/sedona/spark/geopandas/geoseries.py @@ -343,6 +343,7 @@ def __init__( self._col_label: Label self._sindex: SpatialIndex = None self._empty_crs_source: typing.Optional["GeoSeries"] = None + self._empty_crs_value = None if isinstance( data, (GeoDataFrame, GeoSeries, PandasOnSparkSeries, PandasOnSparkDataFrame) @@ -461,6 +462,8 @@ def crs(self) -> Union["CRS", None]: from pyproj import CRS if self._is_empty(): + if self._empty_crs_value is not None: + return self._empty_crs_value if self._empty_crs_source is not None: return self._empty_crs_source.crs return None @@ -482,7 +485,13 @@ def crs(self) -> Union["CRS", None]: srid = 0 if np.isnan(srid) else srid # Sedona returns 0 if SRID doesn't exist. - return CRS.from_user_input(srid) if srid != 0 else None + if srid != 0: + return CRS.from_user_input(srid) + if self._empty_crs_value is not None: + return self._empty_crs_value + if self._empty_crs_source is not None: + return self._empty_crs_source.crs + return None @crs.setter def crs(self, value: Union["CRS", None]): @@ -615,9 +624,15 @@ def set_crs( spark_col = stf.ST_SetSRID(self.spark.column, new_epsg) result = self._query_geometry_column(spark_col, keep_name=True) + result._empty_crs_value = crs + if crs is None: + result._empty_crs_source = None if inplace: self._update_inplace(result, invalidate_sindex=False) + self._empty_crs_value = crs + if crs is None: + self._empty_crs_source = None return None return result diff --git a/python/tests/geopandas/test_spatial_subsetting.py b/python/tests/geopandas/test_spatial_subsetting.py index 6da9b800a5d..5dc66dc14e4 100644 --- a/python/tests/geopandas/test_spatial_subsetting.py +++ b/python/tests/geopandas/test_spatial_subsetting.py @@ -108,6 +108,18 @@ def test_cx_open_reversed_numeric_and_step_slices(self): with pytest.raises(TypeError, match="numeric"): source.cx["left":"right", :] + def test_cx_all_null_series_preserves_crs(self): + expected_source = gpd.GeoSeries([None], name="shape", crs=4326) + source = GeoSeries([None], name="shape", crs=4326) + + result = source.cx[0:1, 0:1] + expected = expected_source.cx[0:1, 0:1] + + self.check_sgpd_equals_gpd(result, expected) + assert len(result) == 0 + assert source.crs == expected_source.crs + assert result.crs == expected.crs + def test_cx_geodataframe_preserves_columns_active_geometry_and_crs(self): index = pd.MultiIndex.from_tuples( [("b", 2), ("a", 2), ("a", 1)], names=["group", "position"] @@ -224,6 +236,18 @@ def test_clip_rectangle_and_empty_results_preserve_crs(self): assert empty_frame.crs == expected_source.crs assert empty_frame.active_geometry_name == "geometry" + def test_clip_all_null_series_preserves_crs(self): + expected_source = gpd.GeoSeries([None], name="shape", crs=4326) + source = GeoSeries([None], name="shape", crs=4326) + + result = source.clip(box(0, 0, 1, 1)) + expected = expected_source.clip(box(0, 0, 1, 1)) + + self.check_sgpd_equals_gpd(result, expected) + assert len(result) == 0 + assert source.crs == expected_source.crs + assert result.crs == expected.crs + def test_clip_geodataframe_preserves_index_columns_and_active_geometry(self): index = pd.MultiIndex.from_tuples( [("b", 2), ("a", 2), ("a", 1), ("c", 1)], From 9796943f496781041f346576349ac65c936f36a7 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Sun, 26 Jul 2026 23:31:47 -0700 Subject: [PATCH 5/8] [GH-3180] Match GeoPandas clip explode semantics --- python/sedona/spark/geopandas/geoseries.py | 7 ++ python/sedona/spark/geopandas/tools/clip.py | 112 +++++++++--------- .../geopandas/test_spatial_subsetting.py | 31 +++++ 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/python/sedona/spark/geopandas/geoseries.py b/python/sedona/spark/geopandas/geoseries.py index 981c13529b7..ce5fd45f5a8 100644 --- a/python/sedona/spark/geopandas/geoseries.py +++ b/python/sedona/spark/geopandas/geoseries.py @@ -343,6 +343,8 @@ def __init__( self._col_label: Label self._sindex: SpatialIndex = None self._empty_crs_source: typing.Optional["GeoSeries"] = None + # Explicit CRS metadata wins over the lineage fallback below when + # geometry rows are empty or carry SRID 0. self._empty_crs_value = None if isinstance( @@ -462,6 +464,8 @@ def crs(self) -> Union["CRS", None]: from pyproj import CRS if self._is_empty(): + # Empty data has no SRID to inspect, so explicit CRS metadata wins + # over the inherited lineage metadata. if self._empty_crs_value is not None: return self._empty_crs_value if self._empty_crs_source is not None: @@ -487,6 +491,9 @@ def crs(self) -> Union["CRS", None]: # Sedona returns 0 if SRID doesn't exist. if srid != 0: return CRS.from_user_input(srid) + # These fallbacks are metadata rather than a fresh read from geometry + # coordinates. Explicit metadata takes precedence over inherited + # lineage metadata, including for non-empty geometries with SRID 0. if self._empty_crs_value is not None: return self._empty_crs_value if self._empty_crs_source is not None: diff --git a/python/sedona/spark/geopandas/tools/clip.py b/python/sedona/spark/geopandas/tools/clip.py index 275b1c6ec8b..e870bcd1196 100644 --- a/python/sedona/spark/geopandas/tools/clip.py +++ b/python/sedona/spark/geopandas/tools/clip.py @@ -87,6 +87,7 @@ def _rebuild_like(obj, internal, sdf): result = GeoDataFrame(PandasOnSparkDataFrame(result_internal)) result._geometry_column_name = obj.active_geometry_name + # Bypass GeoDataFrame.__setattr__, which would interpret this as a new column. object.__setattr__(result, "_empty_crs_source", obj.geometry) return result @@ -326,48 +327,50 @@ def _geometry_family(geometry): ) -def _source_geometry_family(source_sdf, geometry_name): - """Summarize source geometry families without collecting geometries.""" +def _source_geometry_summary(source_sdf, geometry_name): + """Build a one-row source geometry-family summary.""" geometry = scol_for(source_sdf, geometry_name) geometry_type = stf.ST_GeometryType(geometry) family = _geometry_family(geometry) - summary = source_sdf.agg( + return source_sdf.agg( F.max( F.when(geometry_type == _GEOMETRY_COLLECTION, F.lit(1)).otherwise(F.lit(0)) - ).alias("has_collection"), - F.countDistinct(family).alias("family_count"), + ).alias("source_has_collection"), + F.countDistinct(family).alias("source_family_count"), F.min_by(family, scol_for(source_sdf, NATURAL_ORDER_COLUMN_NAME)).alias( - "family" + "source_family" ), - ).first() - return summary.has_collection, summary.family_count, summary.family + ) def _clipped_geometry_summary(sdf, geometry_name): - """Return collection presence and basic family count for clipped rows.""" + """Build a one-row clipped geometry-family summary.""" geometry = scol_for(sdf, geometry_name) geometry_type = stf.ST_GeometryType(geometry) family = _geometry_family(geometry) - summary = sdf.agg( + return sdf.agg( F.max( F.when(geometry_type == _GEOMETRY_COLLECTION, F.lit(1)).otherwise(F.lit(0)) - ).alias("has_collection"), - F.countDistinct(family).alias("family_count"), - ).first() - return summary.has_collection, summary.family_count + ).alias("clipped_has_collection"), + F.countDistinct(family).alias("clipped_family_count"), + ) + + +def _geometry_summaries(source_sdf, clipped_sdf, geometry_name): + """Collect source and clipped type metadata in one Spark action.""" + summary = _source_geometry_summary(source_sdf, geometry_name).crossJoin( + _clipped_geometry_summary(clipped_sdf, geometry_name) + ) + return summary.first() def _keep_only_family(internal, sdf, geometry_name, family): - """Explode new collections and retain only the source geometry family.""" + """Explode multipart rows and retain the source line or polygon family.""" reserved = set(sdf.columns) parent_order_name = _temporary_column_name(sdf, "clip_parent_order", reserved) position_name = _temporary_column_name(sdf, "clip_position", reserved) geometry_value_name = _temporary_column_name(sdf, "clip_geometry", reserved) geometry = scol_for(sdf, geometry_name) - geometry_parts = F.when( - stf.ST_GeometryType(geometry) == _GEOMETRY_COLLECTION, - stf.ST_Dump(geometry), - ).otherwise(F.array(geometry)) expanded = sdf.select( *[ @@ -376,19 +379,24 @@ def _keep_only_family(internal, sdf, geometry_name, family): if name not in (geometry_name, NATURAL_ORDER_COLUMN_NAME) ], scol_for(sdf, NATURAL_ORDER_COLUMN_NAME).alias(parent_order_name), - F.posexplode(geometry_parts).alias(position_name, geometry_value_name), + # GeoPandas explodes the entire clipped result when any new collection + # appears, including sibling MultiPolygon and MultiLineString rows. + F.posexplode(stf.ST_Dump(geometry)).alias(position_name, geometry_value_name), ) allowed_types = { - "point": _POINT_TYPES, "line": _LINE_TYPES, "polygon": _POLYGON_TYPES, - }[family] - expanded = expanded.where( - stf.ST_GeometryType(scol_for(expanded, geometry_value_name)).isin( - *allowed_types + }.get(family) + # GeoPandas filters only line and polygon sources after exploding. Point + # sources retain every exploded part. + if allowed_types is not None: + expanded = expanded.where( + stf.ST_GeometryType(scol_for(expanded, geometry_value_name)).isin( + *allowed_types + ) ) - ).select( + expanded = expanded.select( *[ ( scol_for(expanded, geometry_value_name).alias(geometry_name) @@ -427,7 +435,9 @@ def clip(gdf, mask, keep_geom_type: bool = False, sort: bool = False): ``(minx, miny, maxx, maxy)``. keep_geom_type : bool, default False Retain only the input geometry family when clipping creates lower - dimensional geometries. + dimensional geometries. Matching GeoPandas' warnings and filtering + trigger requires one eager metadata aggregation; geometry rows remain + distributed. sort : bool, default False Return matching rows in their original positional order. @@ -459,27 +469,6 @@ def clip(gdf, mask, keep_geom_type: bool = False, sort: bool = False): internal, source_sdf, geometry_name = _geometry_context(gdf) reserved = set(source_sdf.columns) - source_family = None - if keep_geom_type: - has_collection, family_count, source_family = _source_geometry_family( - source_sdf, geometry_name - ) - if has_collection: - warnings.warn( - "keep_geom_type can not be called on a " - "GeoDataFrame with GeometryCollection.", - UserWarning, - stacklevel=2, - ) - source_family = None - elif family_count > 1: - warnings.warn( - "keep_geom_type can not be called on a mixed type GeoDataFrame.", - UserWarning, - stacklevel=2, - ) - source_family = None - working_sdf, mask_geometry, rectangle = _mask_expression(source_sdf, mask, reserved) source_geometry = scol_for(working_sdf, geometry_name) geometry_type = stf.ST_GeometryType(source_geometry) @@ -503,14 +492,31 @@ def clip(gdf, mask, keep_geom_type: bool = False, sort: bool = False): ~stf.ST_IsEmpty(scol_for(clipped_sdf, geometry_name)) ) - if source_family is not None: - has_new_collection, clipped_family_count = _clipped_geometry_summary( - clipped_sdf, geometry_name - ) + if keep_geom_type: + summary = _geometry_summaries(source_sdf, clipped_sdf, geometry_name) + source_family = summary.source_family + if summary.source_has_collection: + warnings.warn( + "keep_geom_type can not be called on a " + "GeoDataFrame with GeometryCollection.", + UserWarning, + stacklevel=2, + ) + source_family = None + elif summary.source_family_count > 1: + warnings.warn( + "keep_geom_type can not be called on a mixed type GeoDataFrame.", + UserWarning, + stacklevel=2, + ) + source_family = None + # GeoPandas filters only when clipping introduces a collection or an # additional basic family. If every line collapses to a point, for # example, those points remain even with keep_geom_type=True. - if has_new_collection or clipped_family_count > 1: + if source_family is not None and ( + summary.clipped_has_collection or summary.clipped_family_count > 1 + ): clipped_sdf = _keep_only_family( internal, clipped_sdf, geometry_name, source_family ) diff --git a/python/tests/geopandas/test_spatial_subsetting.py b/python/tests/geopandas/test_spatial_subsetting.py index 5dc66dc14e4..5ef1ea28731 100644 --- a/python/tests/geopandas/test_spatial_subsetting.py +++ b/python/tests/geopandas/test_spatial_subsetting.py @@ -27,6 +27,7 @@ GeometryCollection, LineString, MultiPoint, + MultiPolygon, Point, Polygon, box, @@ -333,6 +334,36 @@ def test_clip_keep_geom_type_and_mixed_type_warnings(self): with pytest.warns(UserWarning, match="GeometryCollection"): collection.clip(mask, keep_geom_type=True) + def test_clip_keep_geom_type_explodes_all_multipart_rows(self): + mask = box(0, 0, 2, 2) + expected_source = gpd.GeoSeries( + [ + MultiPolygon( + [ + box(0.5, 0.5, 1, 1), + box(2, 0.5, 3, 1), + ] + ), + MultiPolygon( + [ + box(0.2, 1.2, 0.4, 1.4), + box(0.6, 1.2, 0.8, 1.4), + ] + ), + ], + index=pd.Index(["collection", "multipart"], name="feature_id"), + crs=4326, + ) + source = GeoSeries(expected_source, crs=4326) + + result = source.clip(mask, keep_geom_type=True) + expected = expected_source.clip(mask, keep_geom_type=True) + + self.check_sgpd_equals_gpd(result, expected) + assert list(result.to_geopandas().index) == list(expected.index) + assert list(result.geom_type.to_pandas()) == ["Polygon"] * 3 + self._assert_srid(result, 4326) + def test_clip_keep_geom_type_uses_first_geometry_type_including_null(self): mask = box(0, 0, 1, 1) expected_source = gpd.GeoSeries( From 64507603e571c907a41eebd1bb42fc3504f657b9 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 27 Jul 2026 01:29:10 -0700 Subject: [PATCH 6/8] [GH-3180] Gate clip explosion on new collections --- python/sedona/spark/geopandas/tools/clip.py | 36 ++++++++----- .../geopandas/test_spatial_subsetting.py | 53 +++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/python/sedona/spark/geopandas/tools/clip.py b/python/sedona/spark/geopandas/tools/clip.py index e870bcd1196..ffed1cfb820 100644 --- a/python/sedona/spark/geopandas/tools/clip.py +++ b/python/sedona/spark/geopandas/tools/clip.py @@ -364,13 +364,16 @@ def _geometry_summaries(source_sdf, clipped_sdf, geometry_name): return summary.first() -def _keep_only_family(internal, sdf, geometry_name, family): - """Explode multipart rows and retain the source line or polygon family.""" +def _keep_only_family(internal, sdf, geometry_name, family, explode): + """Retain the source line or polygon family, exploding when requested.""" reserved = set(sdf.columns) parent_order_name = _temporary_column_name(sdf, "clip_parent_order", reserved) position_name = _temporary_column_name(sdf, "clip_position", reserved) geometry_value_name = _temporary_column_name(sdf, "clip_geometry", reserved) geometry = scol_for(sdf, geometry_name) + # GeoPandas explodes the entire result only when clipping introduces a + # GeometryCollection. Multiple basic families alone trigger filtering. + geometry_parts = stf.ST_Dump(geometry) if explode else F.array(geometry) expanded = sdf.select( *[ @@ -379,9 +382,7 @@ def _keep_only_family(internal, sdf, geometry_name, family): if name not in (geometry_name, NATURAL_ORDER_COLUMN_NAME) ], scol_for(sdf, NATURAL_ORDER_COLUMN_NAME).alias(parent_order_name), - # GeoPandas explodes the entire clipped result when any new collection - # appears, including sibling MultiPolygon and MultiLineString rows. - F.posexplode(stf.ST_Dump(geometry)).alias(position_name, geometry_value_name), + F.posexplode(geometry_parts).alias(position_name, geometry_value_name), ) allowed_types = { @@ -409,8 +410,14 @@ def _keep_only_family(internal, sdf, geometry_name, family): scol_for(expanded, parent_order_name), scol_for(expanded, position_name), ) + sort_columns = [scol_for(expanded, parent_order_name)] + if explode: + # GEOS overlay collections place higher-dimensional components first, + # while JTS may emit them in the opposite order. + sort_columns.append(stf.ST_Dimension(scol_for(expanded, geometry_name)).desc()) + sort_columns.append(scol_for(expanded, position_name)) expanded = ( - expanded.orderBy(parent_order_name, position_name) + expanded.orderBy(*sort_columns) .withColumn(NATURAL_ORDER_COLUMN_NAME, F.monotonically_increasing_id()) .drop(parent_order_name, position_name) ) @@ -495,6 +502,7 @@ def clip(gdf, mask, keep_geom_type: bool = False, sort: bool = False): if keep_geom_type: summary = _geometry_summaries(source_sdf, clipped_sdf, geometry_name) source_family = summary.source_family + source_supports_keep_geom_type = True if summary.source_has_collection: warnings.warn( "keep_geom_type can not be called on a " @@ -502,23 +510,27 @@ def clip(gdf, mask, keep_geom_type: bool = False, sort: bool = False): UserWarning, stacklevel=2, ) - source_family = None + source_supports_keep_geom_type = False elif summary.source_family_count > 1: warnings.warn( "keep_geom_type can not be called on a mixed type GeoDataFrame.", UserWarning, stacklevel=2, ) - source_family = None + source_supports_keep_geom_type = False # GeoPandas filters only when clipping introduces a collection or an # additional basic family. If every line collapses to a point, for # example, those points remain even with keep_geom_type=True. - if source_family is not None and ( - summary.clipped_has_collection or summary.clipped_family_count > 1 - ): + explode = bool(summary.clipped_has_collection) + filter_family = source_family is not None and summary.clipped_family_count > 1 + if source_supports_keep_geom_type and (explode or filter_family): clipped_sdf = _keep_only_family( - internal, clipped_sdf, geometry_name, source_family + internal, + clipped_sdf, + geometry_name, + source_family, + explode=explode, ) if sort: diff --git a/python/tests/geopandas/test_spatial_subsetting.py b/python/tests/geopandas/test_spatial_subsetting.py index 5ef1ea28731..dda7ffd7e07 100644 --- a/python/tests/geopandas/test_spatial_subsetting.py +++ b/python/tests/geopandas/test_spatial_subsetting.py @@ -364,6 +364,59 @@ def test_clip_keep_geom_type_explodes_all_multipart_rows(self): assert list(result.geom_type.to_pandas()) == ["Polygon"] * 3 self._assert_srid(result, 4326) + def test_clip_keep_geom_type_preserves_multipart_without_collection(self): + mask = box(0, 0, 2, 2) + expected_source = gpd.GeoSeries( + [ + MultiPolygon( + [ + box(0.2, 0.2, 0.4, 0.4), + box(0.6, 0.6, 0.8, 0.8), + ] + ), + box(2, 0.5, 3, 1), + ], + index=pd.Index(["multi", "edge"], name="feature_id"), + crs=4326, + ) + source = GeoSeries(expected_source, crs=4326) + + result = source.clip(mask, keep_geom_type=True) + expected = expected_source.clip(mask, keep_geom_type=True) + + self.check_sgpd_equals_gpd(result, expected) + assert list(result.to_geopandas().index) == ["multi"] + assert list(result.geom_type.to_pandas()) == ["MultiPolygon"] + self._assert_srid(result, 4326) + + def test_clip_keep_geom_type_explodes_collection_with_null_first_row(self): + mask = box(0, 0, 2, 2) + expected_source = gpd.GeoSeries( + [ + None, + MultiPolygon( + [ + box(0.5, 0.5, 1, 1), + box(2, 0.5, 3, 1), + ] + ), + ], + index=pd.Index(["null", "collection"], name="feature_id"), + crs=4326, + ) + source_seed = expected_source.copy() + source_seed.iloc[0] = box(0, 0, 0.1, 0.1) + source = GeoSeries(source_seed, crs=4326) + source.iloc[0] = None + + result = source.clip(mask, keep_geom_type=True) + expected = expected_source.clip(mask, keep_geom_type=True) + + self.check_sgpd_equals_gpd(result, expected) + assert list(result.to_geopandas().index) == ["collection", "collection"] + assert set(result.geom_type.to_pandas()) == {"Polygon", "LineString"} + self._assert_srid(result, 4326) + def test_clip_keep_geom_type_uses_first_geometry_type_including_null(self): mask = box(0, 0, 1, 1) expected_source = gpd.GeoSeries( From 5d78f89efe4815f3babf628f18c0f26fc5ef87d4 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 27 Jul 2026 10:01:56 -0700 Subject: [PATCH 7/8] [GH-3180] Add legacy GeoPandas spatial index dependency --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index b963f2e5c53..7b1370d4f4c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -71,6 +71,7 @@ dev = [ "pandas>=2.0.0,<3.0.0", "numpy<2", "geopandas", + "rtree; python_version < '3.9'", # https://stackoverflow.com/questions/78949093/how-to-resolve-attributeerror-module-fiona-has-no-attribute-path # cannot set geopandas>=0.14.4 since it doesn't support python 3.8, so we pin fiona to <1.10.0 "fiona<1.10.0", From 470eb140d1ee66d5f84fdb892a8eee5c9951367a Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 27 Jul 2026 12:08:49 -0700 Subject: [PATCH 8/8] [GH-3180] Stabilize legacy clip expectations --- python/tests/geopandas/test_spatial_subsetting.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python/tests/geopandas/test_spatial_subsetting.py b/python/tests/geopandas/test_spatial_subsetting.py index dda7ffd7e07..114cc2badea 100644 --- a/python/tests/geopandas/test_spatial_subsetting.py +++ b/python/tests/geopandas/test_spatial_subsetting.py @@ -358,9 +358,18 @@ def test_clip_keep_geom_type_explodes_all_multipart_rows(self): result = source.clip(mask, keep_geom_type=True) expected = expected_source.clip(mask, keep_geom_type=True) + # GeoPandas 0.13 drops the index name when clip explodes collections. + # Sedona intentionally preserves the source index metadata. + expected.index.names = expected_source.index.names self.check_sgpd_equals_gpd(result, expected) - assert list(result.to_geopandas().index) == list(expected.index) + # Preserve source-row order even though GeoPandas 0.13 places exploded + # multipart rows before the collection row. + assert list(result.to_geopandas().index) == [ + "collection", + "multipart", + "multipart", + ] assert list(result.geom_type.to_pandas()) == ["Polygon"] * 3 self._assert_srid(result, 4326) @@ -411,6 +420,9 @@ def test_clip_keep_geom_type_explodes_collection_with_null_first_row(self): result = source.clip(mask, keep_geom_type=True) expected = expected_source.clip(mask, keep_geom_type=True) + # GeoPandas 0.13 drops the index name when clip explodes collections. + # Sedona intentionally preserves the source index metadata. + expected.index.names = expected_source.index.names self.check_sgpd_equals_gpd(result, expected) assert list(result.to_geopandas().index) == ["collection", "collection"]