Skip to content

Commit ccb3a29

Browse files
authored
feat: Add RESTscan Planning Endpoints and config (#2842)
related to #2775 # Rationale for this change This PR adds some of the rest scanning endpoints needed for the synchronous scan planning And the configuration property used to determine scan planning support. Aligning with the java implementation in apache/iceberg#13400. ## Usage ``` catalog = RestCatalog( "the-best-rest-catalog", uri="http://127.0.0.1:8181", **{"rest.scan-planning.enabled": "true"}, ) ``` Ideally we only would want to use rest scan planning if the catalog is **_capable_**. I remember there was a PR open for this at one point but can't seem to find it. ## Are these changes tested? ## Are there any user-facing changes? new property to configure scanning
1 parent c4a3db3 commit ccb3a29

File tree

2 files changed

+62
-0
lines changed

2 files changed

+62
-0
lines changed

pyiceberg/catalog/rest/__init__.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ class Endpoints:
9696
list_views: str = "namespaces/{namespace}/views"
9797
drop_view: str = "namespaces/{namespace}/views/{view}"
9898
view_exists: str = "namespaces/{namespace}/views/{view}"
99+
plan_table_scan: str = "namespaces/{namespace}/tables/{table}/plan"
100+
fetch_scan_tasks: str = "namespaces/{namespace}/tables/{table}/tasks"
99101

100102

101103
class IdentifierKind(Enum):
@@ -130,6 +132,8 @@ class IdentifierKind(Enum):
130132
SNAPSHOT_LOADING_MODE = "snapshot-loading-mode"
131133
AUTH = "auth"
132134
CUSTOM = "custom"
135+
REST_SCAN_PLANNING_ENABLED = "rest-scan-planning-enabled"
136+
REST_SCAN_PLANNING_ENABLED_DEFAULT = False
133137

134138
NAMESPACE_SEPARATOR = b"\x1f".decode(UTF8)
135139

@@ -269,6 +273,14 @@ def _create_session(self) -> Session:
269273

270274
return session
271275

276+
def is_rest_scan_planning_enabled(self) -> bool:
277+
"""Check if rest server-side scan planning is enabled.
278+
279+
Returns:
280+
True if enabled, False otherwise.
281+
"""
282+
return property_as_bool(self.properties, REST_SCAN_PLANNING_ENABLED, REST_SCAN_PLANNING_ENABLED_DEFAULT)
283+
272284
def _create_legacy_oauth2_auth_manager(self, session: Session) -> AuthManager:
273285
"""Create the LegacyOAuth2AuthManager by fetching required properties.
274286

tests/catalog/test_rest.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1993,3 +1993,53 @@ def test_rest_catalog_context_manager_with_exception_sigv4(self, rest_mock: Mock
19931993

19941994
assert catalog is not None and hasattr(catalog, "_session")
19951995
assert len(catalog._session.adapters) == self.EXPECTED_ADAPTERS_SIGV4
1996+
1997+
def test_rest_scan_planning_disabled_by_default(self, rest_mock: Mocker) -> None:
1998+
rest_mock.get(
1999+
f"{TEST_URI}v1/config",
2000+
json={"defaults": {}, "overrides": {}},
2001+
status_code=200,
2002+
)
2003+
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN)
2004+
2005+
assert catalog.is_rest_scan_planning_enabled() is False
2006+
2007+
def test_rest_scan_planning_enabled_by_property(self, rest_mock: Mocker) -> None:
2008+
rest_mock.get(
2009+
f"{TEST_URI}v1/config",
2010+
json={"defaults": {}, "overrides": {}},
2011+
status_code=200,
2012+
)
2013+
catalog = RestCatalog(
2014+
"rest",
2015+
uri=TEST_URI,
2016+
token=TEST_TOKEN,
2017+
**{"rest-scan-planning-enabled": "true"},
2018+
)
2019+
2020+
assert catalog.is_rest_scan_planning_enabled() is True
2021+
2022+
def test_rest_scan_planning_explicitly_disabled(self, rest_mock: Mocker) -> None:
2023+
rest_mock.get(
2024+
f"{TEST_URI}v1/config",
2025+
json={"defaults": {}, "overrides": {}},
2026+
status_code=200,
2027+
)
2028+
catalog = RestCatalog(
2029+
"rest",
2030+
uri=TEST_URI,
2031+
token=TEST_TOKEN,
2032+
**{"rest-scan-planning-enabled": "false"},
2033+
)
2034+
2035+
assert catalog.is_rest_scan_planning_enabled() is False
2036+
2037+
def test_rest_scan_planning_enabled_from_server_config(self, rest_mock: Mocker) -> None:
2038+
rest_mock.get(
2039+
f"{TEST_URI}v1/config",
2040+
json={"defaults": {"rest-scan-planning-enabled": "true"}, "overrides": {}},
2041+
status_code=200,
2042+
)
2043+
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN)
2044+
2045+
assert catalog.is_rest_scan_planning_enabled() is True

0 commit comments

Comments
 (0)