Update return_type to raise error for metadata source

Ensure return_field_from_args is the only metadata source
by having PySimpleScalarUDF::return_type raise an internal
error. This aligns with DataFusion guidance.

Enhance Python UDF helper documentation to clarify how
callers can declare extension metadata on both arguments
and results.

Author: kosiew

python/datafusion/user_defined.py

@@ -274,9 +274,11 @@ def udf(*args: Any, **kwargs: Any):  # noqa: D417
                 :class:`pyarrow.ChunkedArray` values.
             input_types (list[pa.DataType | pa.Field]): The argument types for ``func``.
                 This list must be of the same length as the number of arguments. Pass
-                :class:`pyarrow.Field` instances to preserve extension metadata.
-            return_type (pa.DataType | pa.Field): The return type of the function. Use a
-                :class:`pyarrow.Field` to preserve metadata on extension arrays.
+                :class:`pyarrow.Field` instances when you need to declare extension
+                metadata for an argument.
+            return_type (pa.DataType | pa.Field): The return type of the function. Supply
+                a :class:`pyarrow.Field` when the result should expose extension metadata
+                to downstream consumers.
             volatility (Volatility | str): See `Volatility` for allowed values.
             name (Optional[str]): A descriptive name for the function.
 

src/udf.rs

@@ -135,7 +135,10 @@ impl ScalarUDFImpl for PySimpleScalarUDF {
     }
 
     fn return_type(&self, _arg_types: &[DataType]) -> datafusion::error::Result<DataType> {
-        Ok(self.return_field.data_type().clone())
+        Err(DataFusionError::Internal(
+            "return_type should be unreachable when return_field_from_args is implemented"
+                .to_string(),
+        ))
     }
 
     fn return_field_from_args(