Add geoarrow dependency and document current capabilities

Author: SaymV

mkdocs/docs/geospatial.md

@@ -0,0 +1,110 @@
+# Geospatial Types
+
+PyIceberg supports Iceberg v3 geospatial primitive types: `geometry` and `geography`.
+
+## Overview
+
+Iceberg v3 introduces native support for spatial data types:
+
+- **`geometry(C)`**: Represents geometric shapes in a coordinate reference system (CRS)
+- **`geography(C, A)`**: Represents geographic shapes with CRS and calculation algorithm
+
+Both types store values as WKB (Well-Known Binary) bytes.
+
+## Requirements
+
+- Iceberg format version 3 or higher
+- `geoarrow-pyarrow` for full GeoArrow extension type support (optional: `pip install pyiceberg[geoarrow]`)
+
+## Usage
+
+### Declaring Columns
+
+```python
+from pyiceberg.schema import Schema
+from pyiceberg.types import NestedField, GeometryType, GeographyType
+
+# Schema with geometry and geography columns
+schema = Schema(
+    NestedField(1, "id", IntegerType(), required=True),
+    NestedField(2, "location", GeometryType(), required=True),
+    NestedField(3, "boundary", GeographyType(), required=False),
+)
+```
+
+### Type Parameters
+
+#### GeometryType
+
+```python
+# Default CRS (OGC:CRS84)
+GeometryType()
+
+# Custom CRS
+GeometryType("EPSG:4326")
+```
+
+#### GeographyType
+
+```python
+# Default CRS (OGC:CRS84) and algorithm (spherical)
+GeographyType()
+
+# Custom CRS
+GeographyType("EPSG:4326")
+
+# Custom CRS and algorithm
+GeographyType("EPSG:4326", "planar")
+```
+
+### String Type Syntax
+
+Types can also be specified as strings in schema definitions:
+
+```python
+# Using string type names
+NestedField(1, "point", "geometry", required=True)
+NestedField(2, "region", "geography", required=True)
+
+# With parameters
+NestedField(3, "location", "geometry('EPSG:4326')", required=True)
+NestedField(4, "boundary", "geography('EPSG:4326', 'planar')", required=True)
+```
+
+## Data Representation
+
+Values are represented as WKB (Well-Known Binary) bytes at runtime:
+
+```python
+# Example: Point(0, 0) in WKB format
+point_wkb = bytes.fromhex("0101000000000000000000000000000000000000")
+```
+
+## Current Limitations
+
+1. **WKB/WKT Conversion**: Converting between WKB bytes and WKT strings requires external libraries (like Shapely). PyIceberg does not include this conversion to avoid heavy dependencies.
+
+2. **Spatial Predicates**: Spatial filtering (e.g., ST_Contains, ST_Intersects) is not yet supported for query pushdown.
+
+3. **Bounds Metrics**: Geometry/geography columns do not currently contribute to data file bounds metrics.
+
+4. **Without geoarrow-pyarrow**: When the `geoarrow-pyarrow` package is not installed, geometry and geography columns are stored as binary without GeoArrow extension type metadata. The Iceberg schema preserves type information, but other tools reading the Parquet files directly may not recognize them as spatial types. Install with `pip install pyiceberg[geoarrow]` for full GeoArrow support.
+
+## Format Version
+
+Geometry and geography types require Iceberg format version 3:
+
+```python
+from pyiceberg.table import TableProperties
+
+# Creating a v3 table
+table = catalog.create_table(
+    identifier="db.spatial_table",
+    schema=schema,
+    properties={
+        TableProperties.FORMAT_VERSION: "3"
+    }
+)
+```
+
+Attempting to use these types with format version 1 or 2 will raise a validation error.

pyiceberg/io/pyarrow.py

@@ -801,24 +801,35 @@ def visit_binary(self, _: BinaryType) -> pa.DataType:
         return pa.large_binary()
 
     def visit_geometry(self, geometry_type: GeometryType) -> pa.DataType:
-        """Convert geometry type to PyArrow binary.
+        """Convert geometry type to PyArrow type.
 
-        Note: PyArrow 21.0.0+ supports native GEOMETRY logical type from Arrow GEO.
-        For now, we use large_binary which stores WKB bytes.
-        Future enhancement: detect PyArrow version and use pa.geometry() when available.
+        When geoarrow-pyarrow is available, returns a GeoArrow WKB extension type
+        with CRS metadata. Otherwise, falls back to large_binary which stores WKB bytes.
         """
-        # TODO: When PyArrow 21.0.0+ is available, use pa.geometry() with CRS metadata
-        return pa.large_binary()
+        try:
+            import geoarrow.pyarrow as ga
+
+            return ga.wkb().with_crs(geometry_type.crs)
+        except ImportError:
+            return pa.large_binary()
 
     def visit_geography(self, geography_type: GeographyType) -> pa.DataType:
-        """Convert geography type to PyArrow binary.
+        """Convert geography type to PyArrow type.
 
-        Note: PyArrow 21.0.0+ supports native GEOGRAPHY logical type from Arrow GEO.
-        For now, we use large_binary which stores WKB bytes.
-        Future enhancement: detect PyArrow version and use pa.geography() when available.
+        When geoarrow-pyarrow is available, returns a GeoArrow WKB extension type
+        with CRS and edge type metadata. Otherwise, falls back to large_binary which stores WKB bytes.
         """
-        # TODO: When PyArrow 21.0.0+ is available, use pa.geography() with CRS and algorithm metadata
-        return pa.large_binary()
+        try:
+            import geoarrow.pyarrow as ga
+
+            wkb_type = ga.wkb().with_crs(geography_type.crs)
+            # Map Iceberg algorithm to GeoArrow edge type
+            if geography_type.algorithm == "spherical":
+                wkb_type = wkb_type.with_edge_type(ga.EdgeType.SPHERICAL)
+            # "planar" is the default edge type in GeoArrow, no need to set explicitly
+            return wkb_type
+        except ImportError:
+            return pa.large_binary()
 
 
 def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar:

pyproject.toml

@@ -96,6 +96,7 @@ hf = ["huggingface-hub>=0.24.0"]
 pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.8.0"]
 datafusion = ["datafusion>=45,<49"]
 gcp-auth = ["google-auth>=2.4.0"]
+geoarrow = ["geoarrow-pyarrow>=0.2.0"]
 
 [dependency-groups]
 dev = [

tests/io/test_pyarrow.py

@@ -98,6 +98,8 @@
     DoubleType,
     FixedType,
     FloatType,
+    GeographyType,
+    GeometryType,
     IntegerType,
     ListType,
     LongType,
@@ -596,6 +598,108 @@ def test_binary_type_to_pyarrow() -> None:
     assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.large_binary()
 
 
+def test_geometry_type_to_pyarrow_without_geoarrow() -> None:
+    """Test geometry type falls back to large_binary when geoarrow is not available."""
+    import sys
+
+    iceberg_type = GeometryType()
+
+    # Remove geoarrow from sys.modules if present and block re-import
+    saved_modules = {}
+    for mod_name in list(sys.modules.keys()):
+        if mod_name.startswith("geoarrow"):
+            saved_modules[mod_name] = sys.modules.pop(mod_name)
+
+    import builtins
+
+    original_import = builtins.__import__
+
+    def mock_import(name: str, *args: Any, **kwargs: Any) -> Any:
+        if name.startswith("geoarrow"):
+            raise ImportError(f"No module named '{name}'")
+        return original_import(name, *args, **kwargs)
+
+    try:
+        builtins.__import__ = mock_import
+        result = visit(iceberg_type, _ConvertToArrowSchema())
+        assert result == pa.large_binary()
+    finally:
+        builtins.__import__ = original_import
+        sys.modules.update(saved_modules)
+
+
+def test_geography_type_to_pyarrow_without_geoarrow() -> None:
+    """Test geography type falls back to large_binary when geoarrow is not available."""
+    import sys
+
+    iceberg_type = GeographyType()
+
+    # Remove geoarrow from sys.modules if present and block re-import
+    saved_modules = {}
+    for mod_name in list(sys.modules.keys()):
+        if mod_name.startswith("geoarrow"):
+            saved_modules[mod_name] = sys.modules.pop(mod_name)
+
+    import builtins
+
+    original_import = builtins.__import__
+
+    def mock_import(name: str, *args: Any, **kwargs: Any) -> Any:
+        if name.startswith("geoarrow"):
+            raise ImportError(f"No module named '{name}'")
+        return original_import(name, *args, **kwargs)
+
+    try:
+        builtins.__import__ = mock_import
+        result = visit(iceberg_type, _ConvertToArrowSchema())
+        assert result == pa.large_binary()
+    finally:
+        builtins.__import__ = original_import
+        sys.modules.update(saved_modules)
+
+
+def test_geometry_type_to_pyarrow_with_geoarrow() -> None:
+    """Test geometry type uses geoarrow WKB extension type when available."""
+    pytest.importorskip("geoarrow.pyarrow")
+    import geoarrow.pyarrow as ga
+
+    # Test default CRS
+    iceberg_type = GeometryType()
+    result = visit(iceberg_type, _ConvertToArrowSchema())
+    expected = ga.wkb().with_crs("OGC:CRS84")
+    assert result == expected
+
+    # Test custom CRS
+    iceberg_type_custom = GeometryType("EPSG:4326")
+    result_custom = visit(iceberg_type_custom, _ConvertToArrowSchema())
+    expected_custom = ga.wkb().with_crs("EPSG:4326")
+    assert result_custom == expected_custom
+
+
+def test_geography_type_to_pyarrow_with_geoarrow() -> None:
+    """Test geography type uses geoarrow WKB extension type with edge type when available."""
+    pytest.importorskip("geoarrow.pyarrow")
+    import geoarrow.pyarrow as ga
+
+    # Test default (spherical algorithm)
+    iceberg_type = GeographyType()
+    result = visit(iceberg_type, _ConvertToArrowSchema())
+    expected = ga.wkb().with_crs("OGC:CRS84").with_edge_type(ga.EdgeType.SPHERICAL)
+    assert result == expected
+
+    # Test custom CRS with spherical
+    iceberg_type_custom = GeographyType("EPSG:4326", "spherical")
+    result_custom = visit(iceberg_type_custom, _ConvertToArrowSchema())
+    expected_custom = ga.wkb().with_crs("EPSG:4326").with_edge_type(ga.EdgeType.SPHERICAL)
+    assert result_custom == expected_custom
+
+    # Test planar algorithm (no edge type set, uses default)
+    iceberg_type_planar = GeographyType("OGC:CRS84", "planar")
+    result_planar = visit(iceberg_type_planar, _ConvertToArrowSchema())
+    expected_planar = ga.wkb().with_crs("OGC:CRS84")
+    assert result_planar == expected_planar
+
+
 def test_struct_type_to_pyarrow(table_schema_simple: Schema) -> None:
     expected = pa.struct(
         [