More user documentation

Author: timsaucer

python/datafusion/user_defined.py

@@ -298,7 +298,16 @@ class Accumulator(metaclass=ABCMeta):
 
     @abstractmethod
     def state(self) -> list[pa.Scalar]:
-        """Return the current state."""
+        """Return the current state.
+
+        While this function template expects PyArrow Scalar values return type,
+        you can return any value that can be converted into a Scalar. This
+        includes basic Python data types such as integers and strings. In
+        addition to primitive types, we currently support PyArrow, nanoarrow,
+        and arro3 objects in addition to primitive data types. Other objects
+        that support the Arrow FFI standard will be given a "best attempt" at
+        conversion to scalar objects.
+        """
 
     @abstractmethod
     def update(self, *values: pa.Array) -> None:
@@ -312,18 +321,13 @@ def merge(self, states: list[pa.Array]) -> None:
     def evaluate(self) -> pa.Scalar:
         """Return the resultant value.
 
-        If you need to return a list, wrap it in a scalar with the correct
-        list type, for example::
-
-            import pyarrow as pa
-
-            return pa.scalar(
-                [pa.scalar("2024-01-01T00:00:00Z")],
-                type=pa.list_(pa.timestamp("ms")),
-            )
-
-        Returning a ``pyarrow.Array`` from ``evaluate`` is not supported unless
-        you explicitly convert it to a list-valued scalar.
+        While this function template expects a PyArrow Scalar value return type,
+        you can return any value that can be converted into a Scalar. This
+        includes basic Python data types such as integers and strings. In
+        addition to primitive types, we currently support PyArrow, nanoarrow,
+        and arro3 objects in addition to primitive data types. Other objects
+        that support the Arrow FFI standard will be given a "best attempt" at
+        conversion to scalar objects.
         """
 
 

src/common/data_type.rs

@@ -22,6 +22,9 @@ use datafusion::logical_expr::expr::NullTreatment as DFNullTreatment;
 use pyo3::exceptions::{PyNotImplementedError, PyValueError};
 use pyo3::prelude::*;
 
+/// A [`ScalarValue`] wrapped in a Python object. This struct allows for conversion
+/// from a variety of Python objects into a [`ScalarValue`]. See
+/// ``FromPyArrow::from_pyarrow_bound`` conversion details.
 #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)]
 pub struct PyScalarValue(pub ScalarValue);
 

src/pyarrow_util.rs

@@ -31,6 +31,9 @@ use pyo3::{Bound, FromPyObject, PyAny, PyResult, Python};
 use crate::common::data_type::PyScalarValue;
 use crate::errors::PyDataFusionError;
 
+/// Helper function to turn an Array into a ScalarValue. If ``as_list_array`` is true,
+/// the array will be turned into a ``ListArray``. Otherwise, we extract the first value
+/// from the array.
 fn array_to_scalar_value(array: ArrayRef, as_list_array: bool) -> PyResult<PyScalarValue> {
     if as_list_array {
         let field = Arc::new(Field::new_list_field(
@@ -46,6 +49,10 @@ fn array_to_scalar_value(array: ArrayRef, as_list_array: bool) -> PyResult<PySca
     }
 }
 
+/// Helper function to take any Python object that contains an Arrow PyCapsule
+/// interface and attempt to extract a scalar value from it. If `as_list_array`
+/// is true, the array will be turned into a ``ListArray``. Otherwise, we extract
+/// the first value from the array.
 fn pyobj_extract_scalar_via_capsule(
     value: &Bound<'_, PyAny>,
     as_list_array: bool,