feat: update SessionContext to support dynamic auto-registration of Python objects based on session config

Author: kosiew

docs/source/user-guide/dataframe/index.rst

@@ -228,10 +228,10 @@ Core Classes
     * :py:meth:`~datafusion.SessionContext.from_pandas` - Create from Pandas DataFrame
     * :py:meth:`~datafusion.SessionContext.from_arrow` - Create from Arrow data
 
-    ``SessionContext`` automatically resolves SQL table names that match
-    in-scope Python data objects. When ``auto_register_python_objects`` is
-    enabled (the default), a query such as ``ctx.sql("SELECT * FROM pdf")``
-    will register a pandas or PyArrow object named ``pdf`` without calling
+    ``SessionContext`` can automatically resolve SQL table names that match
+    in-scope Python data objects. When automatic lookup is enabled, a query
+    such as ``ctx.sql("SELECT * FROM pdf")`` will register a pandas or
+    PyArrow object named ``pdf`` without calling
     :py:meth:`~datafusion.SessionContext.from_pandas` or
     :py:meth:`~datafusion.SessionContext.from_arrow` explicitly. This requires
     the corresponding library (``pandas`` for pandas objects, ``pyarrow`` for
@@ -242,16 +242,18 @@ Core Classes
         import pandas as pd
         from datafusion import SessionContext
 
-        ctx = SessionContext()
+        ctx = SessionContext(auto_register_python_objects=True)
         pdf = pd.DataFrame({"value": [1, 2, 3]})
 
         df = ctx.sql("SELECT SUM(value) AS total FROM pdf")
         print(df.to_pandas())  # automatically registers `pdf`
 
-    To opt out, either pass ``auto_register_python_objects=False`` when
-    constructing the session, or call
-    :py:meth:`~datafusion.SessionContext.set_python_table_lookup` with
-    ``False`` to require explicit registration.
+    Automatic lookup is disabled by default. Enable it by passing
+    ``auto_register_python_objects=True`` when constructing the session or by
+    configuring :py:class:`~datafusion.SessionConfig` with
+    :py:meth:`~datafusion.SessionConfig.with_python_table_lookup`. Use
+    :py:meth:`~datafusion.SessionContext.set_python_table_lookup` to toggle the
+    behaviour at runtime.
 
     See: :py:class:`datafusion.SessionContext`
 

python/datafusion/context.py

@@ -503,7 +503,7 @@ def __init__(
         config: SessionConfig | None = None,
         runtime: RuntimeEnvBuilder | None = None,
         *,
-        auto_register_python_objects: bool = True,
+        auto_register_python_objects: bool | None = None,
     ) -> None:
         """Main interface for executing queries with DataFusion.
 
@@ -516,7 +516,10 @@ def __init__(
             runtime: Runtime configuration options.
             auto_register_python_objects: Automatically register referenced
                 Python objects (such as pandas or PyArrow data) when ``sql``
-                queries reference them by name.
+                queries reference them by name. When omitted, this defaults to
+                the value configured via
+                :py:meth:`~datafusion.SessionConfig.with_python_table_lookup`
+                (``False`` unless explicitly enabled).
 
         Example usage:
 
@@ -532,6 +535,12 @@ def __init__(
             config.config_internal if config is not None else None,
             runtime.config_internal if runtime is not None else None,
         )
+
+        if auto_register_python_objects is None:
+            auto_register_python_objects = getattr(
+                config, "_python_table_lookup", False
+            )
+
         self._auto_python_table_lookup = auto_register_python_objects
 
     def __repr__(self) -> str:
@@ -560,18 +569,18 @@ def enable_url_table(self) -> SessionContext:
         obj = klass.__new__(klass)
         obj.ctx = self.ctx.enable_url_table()
         obj._auto_python_table_lookup = getattr(
-            self, "_auto_python_table_lookup", True
+            self, "_auto_python_table_lookup", False
         )
         return obj
 
     def set_python_table_lookup(self, enabled: bool = True) -> SessionContext:
         """Enable or disable automatic registration of Python objects in SQL.
 
         Args:
-            enabled: When ``True`` (default), SQL queries automatically attempt
-                to resolve missing table names by looking up Python objects in
-                the caller's scope. When ``False``, missing tables will raise an
-                error unless they have been explicitly registered.
+            enabled: When ``True``, SQL queries automatically attempt to
+                resolve missing table names by looking up Python objects in the
+                caller's scope. Use ``False`` to require explicit registration
+                of any referenced tables.
 
         Returns:
             The current :py:class:`SessionContext` instance for chaining.
@@ -653,7 +662,7 @@ def _execute_sql() -> DataFrame:
         try:
             return _execute_sql()
         except Exception as err:
-            if not getattr(self, "_auto_python_table_lookup", True):
+            if not getattr(self, "_auto_python_table_lookup", False):
                 raise
 
             missing_tables = self._extract_missing_table_names(err)

python/tests/test_context.py

@@ -738,10 +738,10 @@ def test_sql_with_options_no_statements(ctx):
         ctx.sql_with_options(sql, options=options)
 
 
-def test_sql_auto_register_pandas():
+def test_session_config_python_table_lookup_enables_auto_registration():
     pd = pytest.importorskip("pandas")
 
-    ctx = SessionContext()
+    ctx = SessionContext(config=SessionConfig().with_python_table_lookup(True))
     pdf = pd.DataFrame({"value": [1, 2, 3]})
     assert len(pdf) == 3
 
@@ -750,7 +750,7 @@ def test_sql_auto_register_pandas():
 
 
 def test_sql_auto_register_arrow():
-    ctx = SessionContext()
+    ctx = SessionContext(auto_register_python_objects=True)
     arrow_table = pa.table({"value": [1, 2, 3, 4]})
     assert arrow_table.num_rows == 4
 
@@ -761,7 +761,7 @@ def test_sql_auto_register_arrow():
 def test_sql_auto_register_disabled():
     pd = pytest.importorskip("pandas")
 
-    ctx = SessionContext(auto_register_python_objects=False)
+    ctx = SessionContext()
     pdf = pd.DataFrame({"value": [1, 2, 3]})
     assert len(pdf) == 3