refactor: enhance autoapi_skip_member_fn to skip private variables and avoid documentation duplication

Author: kosiew

docs/source/conf.py

@@ -83,6 +83,9 @@ def autoapi_skip_member_fn(app, what, name, obj, skip, options) -> bool:  # noqa
         # Duplicate modules (skip module-level docs to avoid duplication)
         ("module", "datafusion.col"),
         ("module", "datafusion.udf"),
+        # Private variables causing duplicate documentation
+        ("data", "datafusion.utils._PYARROW_DATASET_TYPES"),
+        ("variable", "datafusion.utils._PYARROW_DATASET_TYPES"),
         # Deprecated
         ("class", "datafusion.substrait.serde"),
         ("class", "datafusion.substrait.plan"),
@@ -94,6 +97,10 @@ def autoapi_skip_member_fn(app, what, name, obj, skip, options) -> bool:  # noqa
     if (what, name) in skip_contents:
         skip = True
 
+    # Skip private members that start with underscore to avoid duplication
+    if name.split(".")[-1].startswith("_") and what in ("data", "variable"):
+        skip = True
+
     return skip