Add Rust helper for minimal in-memory table provider

This commit introduces a new Rust-side helper function,
`make_table_provider_capsule`, which constructs a minimal
in-memory table provider capsule and registers it with the
internal catalog module. The helper is now exposed via
`datafusion.catalog.make_table_provider_capsule()`.

Additionally, the user guide has been updated to emphasize
the destructor requirement for externally supplied capsules.
A revised PyCapsule regression example has been included,
along with a regression test to ensure the new helper
correctly interacts with `SessionContext.read_table`.

Author: kosiew

docs/source/user-guide/io/table_provider.rst

@@ -28,6 +28,15 @@ via `PyCapsule <https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule>`_.
 
 A complete example can be found in the `examples folder <https://github.com/apache/datafusion-python/tree/main/examples>`_.
 
+.. note::
+
+   DataFusion validates that table-provider capsules were created through the
+   ``datafusion_ffi`` helpers so that the release callback is installed. Use a
+   helper such as :func:`datafusion.catalog.make_table_provider_capsule` (for an
+   empty in-memory provider) or the constructors provided by :mod:`datafusion_ffi`
+   when fabricating capsules for tests and examples instead of calling
+   ``PyCapsule_New`` directly.
+
 .. code-block:: rust
 
     #[pymethods]

examples/pycapsule_failure.py

@@ -1,42 +1,53 @@
+"""Demonstrate the destructor requirement for table-provider PyCapsules.
+
+DataFusion rejects table-provider capsules that are fabricated directly via
+``PyCapsule_New`` because they lack the release hook installed by the
+``datafusion_ffi`` helpers. Use :func:`datafusion.catalog.make_table_provider_capsule`
+or the Rust-side constructors exposed by :mod:`datafusion_ffi` to obtain a valid
+capsule instead of constructing one manually.
+"""
+
 from __future__ import annotations
 
 import ctypes
 
 from datafusion import SessionContext, Table
+from datafusion.catalog import make_table_provider_capsule
+
+_PYCAPSULE_NEW = ctypes.pythonapi.PyCapsule_New
+_PYCAPSULE_NEW.restype = ctypes.py_object
+_PYCAPSULE_NEW.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p]
 
 
 # Keep the backing memory alive for the lifetime of the module so the capsule
 # always wraps a valid (non-null) pointer. The capsule content is irrelevant for
 # this regression example—we only need a non-null address.
 _DUMMY_CAPSULE_BYTES = ctypes.create_string_buffer(b"x")
 
-class CapsuleContainer:
-    def __init__(self):
-        self.__datafusion_table_provider__ = make_table_provider_capsule
 
-def make_table_provider_capsule() -> object:
-    """Create a dummy PyCapsule with the expected table provider name."""
+def make_invalid_capsule() -> object:
+    """Create an intentionally invalid PyCapsule without a destructor."""
 
-    pycapsule_new = ctypes.pythonapi.PyCapsule_New
-    pycapsule_new.restype = ctypes.py_object
-    pycapsule_new.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p]
     dummy_ptr = ctypes.cast(_DUMMY_CAPSULE_BYTES, ctypes.c_void_p)
-    return pycapsule_new(dummy_ptr, b"datafusion_table_provider", None)
+    return _PYCAPSULE_NEW(dummy_ptr, b"datafusion_table_provider", None)
 
 
 def main() -> None:
-    """Attempt to use the capsule the same way existing callers do."""
+    """Showcase both the failure and the supported code paths."""
 
     ctx = SessionContext()
+
     try:
-        capsule = CapsuleContainer()
-    except Exception as err:
-        print("Creating the PyCapsule failed:", err)
-        return
+        ctx.read_table(make_invalid_capsule())
+    except ValueError as err:
+        print("Creating the PyCapsule failed, as expected:", err)
+
+    valid_capsule = make_table_provider_capsule()
+    ctx.read_table(valid_capsule)
 
+    valid_capsule_for_table = make_table_provider_capsule()
+    Table.from_table_provider_capsule(valid_capsule_for_table)
 
-    ctx.read_table(capsule)
-    
 
 if __name__ == "__main__":
-    main()
+    main()

python/datafusion/catalog.py

@@ -42,6 +42,7 @@
     "Schema",
     "SchemaProvider",
     "Table",
+    "make_table_provider_capsule",
 ]
 
 
@@ -182,9 +183,7 @@ def from_dataset(dataset: pa.dataset.Dataset) -> Table:
     def from_table_provider_capsule(capsule: object) -> Table:
         """Wrap a validated table provider :class:`PyCapsule` as a :class:`Table`."""
 
-        return Table(
-            df_internal.catalog.RawTable.from_table_provider_capsule(capsule)
-        )
+        return Table(df_internal.catalog.RawTable.from_table_provider_capsule(capsule))
 
     @property
     def schema(self) -> pa.Schema:
@@ -197,6 +196,18 @@ def kind(self) -> str:
         return self._inner.kind
 
 
+def make_table_provider_capsule() -> object:
+    """Return a minimal table-provider capsule with a valid release hook.
+
+    DataFusion validates that table-provider capsules originate from
+    :mod:`datafusion_ffi` helpers so the release callback is registered. This
+    helper exposes that functionality to Python callers by fabricating an empty
+    in-memory table provider using :mod:`datafusion_ffi` under the hood.
+    """
+
+    return df_internal.catalog.make_table_provider_capsule()
+
+
 class CatalogProvider(ABC):
     """Abstract class for defining a Python based Catalog Provider."""
 

python/tests/test_context.py

@@ -32,6 +32,7 @@
     column,
     literal,
 )
+from datafusion.catalog import make_table_provider_capsule
 
 _PYCAPSULE_NEW = ctypes.pythonapi.PyCapsule_New
 _PYCAPSULE_NEW.restype = ctypes.py_object
@@ -372,23 +373,29 @@ def __datafusion_table_provider__(self) -> object:
         ctx.read_table(container)
 
     with pytest.raises(ValueError, match="missing a destructor"):
-        Table.from_table_provider_capsule(
-            container.__datafusion_table_provider__()
-        )
+        Table.from_table_provider_capsule(container.__datafusion_table_provider__())
 
 
 def test_read_table_with_raw_table_provider_capsule(ctx):
     df_internal = pytest.importorskip("datafusion._internal")
-    assert hasattr(
-        df_internal.catalog.RawTable, "from_table_provider_capsule"
-    )
+    assert hasattr(df_internal.catalog.RawTable, "from_table_provider_capsule")
 
     capsule, _backing = _make_invalid_table_provider_capsule()
 
     with pytest.raises(ValueError, match="missing a destructor"):
         ctx.read_table(capsule)
 
 
+def test_read_table_accepts_valid_table_provider_capsule(ctx):
+    capsule = make_table_provider_capsule()
+    result = ctx.read_table(capsule).collect()
+    assert result == []
+
+    table_capsule = make_table_provider_capsule()
+    table = Table.from_table_provider_capsule(table_capsule)
+    assert table.kind in {"physical", "view", "temporary"}
+
+
 def test_deregister_table(ctx, database):
     default = ctx.catalog()
     public = default.schema("public")

src/catalog.rs

@@ -19,20 +19,24 @@ use crate::dataset::Dataset;
 use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionError, PyDataFusionResult};
 use crate::table::PyTable;
 use crate::utils::{validate_pycapsule, wait_for_future};
+use arrow_array::RecordBatch;
+use arrow_schema::Schema;
 use async_trait::async_trait;
 use datafusion::catalog::{MemoryCatalogProvider, MemorySchemaProvider};
 use datafusion::common::DataFusionError;
 use datafusion::{
     catalog::{CatalogProvider, SchemaProvider},
-    datasource::TableProvider,
+    datasource::{MemTable, TableProvider},
 };
 use datafusion_ffi::schema_provider::{FFI_SchemaProvider, ForeignSchemaProvider};
+use datafusion_ffi::table_provider::FFI_TableProvider;
 use pyo3::exceptions::PyKeyError;
 use pyo3::prelude::*;
 use pyo3::types::PyCapsule;
 use pyo3::IntoPyObjectExt;
 use std::any::Any;
 use std::collections::HashSet;
+use std::ffi::CString;
 use std::sync::Arc;
 
 #[pyclass(name = "RawCatalog", module = "datafusion.catalog", subclass)]
@@ -452,10 +456,22 @@ impl CatalogProvider for RustWrappedPyCatalogProvider {
     }
 }
 
+#[pyfunction]
+pub fn make_table_provider_capsule(py: Python<'_>) -> PyResult<Bound<'_, PyCapsule>> {
+    let schema = Arc::new(Schema::new(vec![]));
+    let batch = RecordBatch::new_empty(Arc::clone(&schema));
+    let provider = MemTable::try_new(schema, vec![vec![batch]]).map_err(py_datafusion_err)?;
+    let provider = FFI_TableProvider::new(Arc::new(provider), false, None);
+    let name = CString::new("datafusion_table_provider").expect("static capsule name");
+
+    PyCapsule::new(py, provider, Some(name))
+}
+
 pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyCatalog>()?;
     m.add_class::<PySchema>()?;
     m.add_class::<PyTable>()?;
+    m.add_function(pyo3::wrap_pyfunction!(make_table_provider_capsule, m)?)?;
 
     Ok(())
 }