actually get linting passing and bring up to date with new catalog

Author: jayceslesar

pyiceberg/catalog/__init__.py

@@ -22,13 +22,12 @@
 import re
 import uuid
 from abc import ABC, abstractmethod
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
 from dataclasses import dataclass
 from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
-    Iterator,
     cast,
 )
 

pyiceberg/catalog/bigquery_metastore.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import json
+from collections.abc import Iterator
 from typing import TYPE_CHECKING, Any, Union
 
 from google.api_core.exceptions import NotFound
@@ -244,7 +245,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None:
         except NotFound as e:
             raise NoSuchNamespaceError(f"Namespace {namespace} does not exist.") from e
 
-    def list_tables(self, namespace: str | Identifier) -> list[Identifier]:
+    def list_tables(self, namespace: str | Identifier) -> Iterator[Identifier]:
         database_name = self.identifier_to_database(namespace)
         iceberg_tables: list[Identifier] = []
         try:
@@ -256,17 +257,17 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]:
                 iceberg_tables.append((database_name, bq_table_list_item.table_id))
         except NotFound:
             raise NoSuchNamespaceError(f"Namespace (dataset) '{database_name}' not found.") from None
-        return iceberg_tables
+        yield from iceberg_tables
 
-    def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]:
+    def list_namespaces(self, namespace: str | Identifier = ()) -> Iterator[Identifier]:
         # Since this catalog only supports one-level namespaces, it always returns an empty list unless
         # passed an empty namespace to list all namespaces within the catalog.
         if namespace:
             raise NoSuchNamespaceError(f"Namespace (dataset) '{namespace}' not found.") from None
 
         # List top-level datasets
         datasets_iterator = self.client.list_datasets()
-        return [(dataset.dataset_id,) for dataset in datasets_iterator]
+        yield from ((dataset.dataset_id,) for dataset in datasets_iterator)
 
     def register_table(self, identifier: str | Identifier, metadata_location: str) -> Table:
         """Register a new table using existing metadata.
@@ -299,7 +300,7 @@ def register_table(self, identifier: str | Identifier, metadata_location: str) -
 
         return self.load_table(identifier=identifier)
 
-    def list_views(self, namespace: str | Identifier) -> list[Identifier]:
+    def list_views(self, namespace: str | Identifier) -> Iterator[Identifier]:
         raise NotImplementedError
 
     def drop_view(self, identifier: str | Identifier) -> None:

pyiceberg/catalog/dynamodb.py

@@ -15,11 +15,11 @@
 #  specific language governing permissions and limitations
 #  under the License.
 import uuid
+from collections.abc import Iterator
 from time import time
 from typing import (
     TYPE_CHECKING,
     Any,
-    Iterator,
     Optional,
     Union,
 )
@@ -438,8 +438,6 @@ def list_tables(self, namespace: str | Identifier) -> Iterator[Identifier]:
 
                 yield self.identifier_to_tuple(identifier_col)
 
-        return table_identifiers
-
     def list_namespaces(self, namespace: str | Identifier = ()) -> Iterator[Identifier]:
         """List top-level namespaces from the catalog.
 

pyiceberg/catalog/glue.py

@@ -16,10 +16,10 @@
 #  under the License.
 
 
+from collections.abc import Iterator
 from typing import (
     TYPE_CHECKING,
     Any,
-    Iterator,
     Optional,
     Union,
     cast,

pyiceberg/catalog/hive.py

@@ -18,11 +18,11 @@
 import logging
 import socket
 import time
+from collections.abc import Iterator
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
     Any,
-    Iterator,
     Union,
 )
 from urllib.parse import urlparse

pyiceberg/catalog/noop.py

@@ -14,9 +14,9 @@
 #  KIND, either express or implied.  See the License for the
 #  specific language governing permissions and limitations
 #  under the License.
+from collections.abc import Iterator
 from typing import (
     TYPE_CHECKING,
-    Iterator,
     Union,
 )
 

pyiceberg/catalog/rest/__init__.py

@@ -14,11 +14,11 @@
 #  KIND, either express or implied.  See the License for the
 #  specific language governing permissions and limitations
 #  under the License.
+from collections.abc import Iterator
 from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
-    Iterator,
     Union,
 )
 
@@ -599,7 +599,7 @@ def register_table(self, identifier: str | Identifier, metadata_location: str) -
         return self._response_to_table(self.identifier_to_tuple(identifier), table_response)
 
     @retry(**_RETRY_ARGS)
-    def list_tables(self, namespace: str | Identifier) -> list[Identifier]:
+    def _fetch_tables(self, namespace: str | Identifier) -> list[Identifier]:
         namespace_tuple = self._check_valid_namespace_identifier(namespace)
         namespace_concat = NAMESPACE_SEPARATOR.join(namespace_tuple)
         response = self._session.get(self.url(Endpoints.list_tables, namespace=namespace_concat))
@@ -609,7 +609,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]:
             _handle_non_200_response(exc, {404: NoSuchNamespaceError})
         return [(*table.namespace, table.name) for table in ListTablesResponse.model_validate_json(response.text).identifiers]
 
-    def list_tables(self, namespace: Union[str, Identifier]) -> Iterator[Identifier]:
+    def list_tables(self, namespace: str | Identifier) -> Iterator[Identifier]:
         """List tables, returning an iterator to handle retry logic properly."""
         tables = self._fetch_tables(namespace)
         yield from tables
@@ -685,7 +685,7 @@ def _remove_catalog_name_from_table_request_identifier(self, table_request: Comm
         return table_request
 
     @retry(**_RETRY_ARGS)
-    def list_views(self, namespace: str | Identifier) -> list[Identifier]:
+    def _fetch_views(self, namespace: str | Identifier) -> list[Identifier]:
         namespace_tuple = self._check_valid_namespace_identifier(namespace)
         namespace_concat = NAMESPACE_SEPARATOR.join(namespace_tuple)
         response = self._session.get(self.url(Endpoints.list_views, namespace=namespace_concat))
@@ -695,7 +695,7 @@ def list_views(self, namespace: str | Identifier) -> list[Identifier]:
             _handle_non_200_response(exc, {404: NoSuchNamespaceError})
         return [(*view.namespace, view.name) for view in ListViewsResponse.model_validate_json(response.text).identifiers]
 
-    def list_views(self, namespace: Union[str, Identifier]) -> Iterator[Identifier]:
+    def list_views(self, namespace: str | Identifier) -> Iterator[Identifier]:
         """List views, returning an iterator to handle retry logic properly."""
         yield from self._fetch_views(namespace)
 
@@ -766,7 +766,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None:
             _handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: NamespaceNotEmptyError})
 
     @retry(**_RETRY_ARGS)
-    def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]:
+    def _fetch_namespaces(self, namespace: str | Identifier) -> list[Identifier]:
         namespace_tuple = self.identifier_to_tuple(namespace)
         response = self._session.get(
             self.url(
@@ -782,7 +782,7 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]:
 
         return ListNamespaceResponse.model_validate_json(response.text).namespaces
 
-    def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> Iterator[Identifier]:
+    def list_namespaces(self, namespace: str | Identifier = ()) -> Iterator[Identifier]:
         """List namespaces, returning an iterator to handle retry logic properly."""
         yield from self._fetch_namespaces(namespace)
 

pyiceberg/catalog/sql.py

@@ -15,9 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections.abc import Iterator
 from typing import (
     TYPE_CHECKING,
-    Iterator,
     Union,
 )
 
@@ -601,6 +601,7 @@ def list_tables(self, namespace: str | Identifier) -> Iterator[Identifier]:
         with Session(self.engine) as session:
             result = session.scalars(stmt)
             identifiers = [(Catalog.identifier_to_tuple(table.table_namespace) + (table.table_name,)) for table in result]
+            yield from identifiers
 
     def list_namespaces(self, namespace: str | Identifier = ()) -> Iterator[Identifier]:
         """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog.

pyiceberg/cli/output.py

@@ -16,8 +16,8 @@
 # under the License.
 import json
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
 from typing import (
-    Iterator,
     Any,
 )
 from uuid import UUID

tests/integration/test_catalog.py

@@ -176,7 +176,7 @@ def test_list_tables(test_catalog: Catalog, table_schema_nested: Schema, databas
     for table_name in table_list:
         test_catalog.create_table((database_name, table_name), table_schema_nested)
     identifier_list = test_catalog.list_tables(database_name)
-    assert len(identifier_list) == len(table_list)
+    assert len(list(identifier_list)) == len(table_list)
     for table_name in table_list:
         assert (database_name, table_name) in identifier_list
 
@@ -449,7 +449,7 @@ def test_list_namespaces(test_catalog: Catalog, database_list: list[str]) -> Non
     db_list = test_catalog.list_namespaces()
     for database_name in database_list:
         assert (database_name,) in db_list
-    assert len(test_catalog.list_namespaces(list(database_list)[0])) == 0
+    assert len(list(test_catalog.list_namespaces(list(database_list)[0]))) == 0
 
 
 @pytest.mark.integration