From 0e9c5f95316c6d0a29ecbfc120266fdb5880f0b2 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 7 Feb 2025 11:41:52 +0000 Subject: [PATCH 01/27] feat: add pagination interfaces --- ibmcloudant/__init__.py | 1 + ibmcloudant/features/pagination.py | 231 +++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 ibmcloudant/features/pagination.py diff --git a/ibmcloudant/__init__.py b/ibmcloudant/__init__.py index 650d39da..0f2c3f75 100644 --- a/ibmcloudant/__init__.py +++ b/ibmcloudant/__init__.py @@ -23,6 +23,7 @@ from .couchdb_session_token_manager import CouchDbSessionTokenManager from .cloudant_v1 import CloudantV1 from .features.changes_follower import ChangesFollower +from .features.pagination import Pager, PagerType # sdk-core's __construct_authenticator works with a long switch-case so monkey-patching is required get_authenticator.__construct_authenticator = new_construct_authenticator diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py new file mode 100644 index 00000000..285afa2d --- /dev/null +++ b/ibmcloudant/features/pagination.py @@ -0,0 +1,231 @@ +# coding: utf-8 +# © Copyright IBM Corporation 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + Feature for paginating requests. + + Import :class:`~ibmcloudant.Pager` and :class:`~ibmcloudant.PagerType` + from :mod:`ibmcloudant`. + Use :meth:`Pager.new_pager` to create a :class:`Pager` for different + :class:`PagerType` operations. +""" +from abc import abstractmethod +from collections.abc import Callable +from enum import auto, Enum +from functools import partial +from typing import Generic, Protocol, TypeVar + +from ibm_cloud_sdk_core import DetailedResponse +from ibmcloudant.cloudant_v1 import CloudantV1,\ + AllDocsResult, DocsResultRow, Document, FindResult, SearchResult, SearchResultRow, ViewResult, ViewResultRow + +# Type variable for the result +R = TypeVar('R', AllDocsResult, FindResult, SearchResult, ViewResult) +# Type variable for the items +I = TypeVar('I', DocsResultRow, Document, SearchResultRow, ViewResultRow) +# Type variable for the key in key based paging +K = TypeVar('K') + +class PagerType(Enum): + """ + Enumeration of the available Pager types + """ + POST_ALL_DOCS = auto() + POST_DESIGN_DOCS = auto() + POST_FIND = auto() + POST_PARTITION_ALL_DOCS = auto() + POST_PARTITION_FIND = auto() + POST_PARTITION_SEARCH = auto() + POST_PARTITION_VIEW = auto() + POST_SEARCH = auto() + POST_VIEW = auto() + +class Pager(Protocol[R, I]): + """ + Protocol for pagination of Cloudant operations. + Use Pager.new_pager to create a new pager for one of + the operation types in PagerType. + """ + + def has_next(self) -> bool: + """ + returns True if there may be another page of results, False otherwise + """ + pass + + @abstractmethod + def get_next(self) -> list[I]: + """ + returns the next page of results + """ + pass + + def get_all(self) -> list[I]: + """ + returns all the pages of results in single list + """ + pass + + @classmethod + def new_pager(cls, client:CloudantV1, type: PagerType, **kwargs,): + """ + Create a new Pager. + client: CloudantV1 - the Cloudant service client + type: PagerType - the operation type to paginate + kwargs: dict - the options for the operation + """ + pass + +class _BasePager(Pager): + + def __init__(self, + client: CloudantV1, + operation: Callable[..., DetailedResponse], + page_opts: list[str], + opts:dict): + self._client = client + # TODO split the opts into fixed and page parts based on page_opts + self._next_page_opts = {} + fixed_opts = {} + # Partial method with the fixed ops + self._next_request_function = partial(operation, **fixed_opts) + + def _next_request(self) -> R: + response: DetailedResponse = self._next_request_function(**self._next_page_opts) + result: dict = response.get_result() + return self._result_converter()(result) + + @abstractmethod + def _result_converter(self) -> Callable[[dict], R]: + ... + + @abstractmethod + def _items(self, result: R) -> list[I]: + ... + + @abstractmethod + def _get_next_page_options(self, result: R) -> dict: + ... + +class _KeyPager(_BasePager, Generic[K]): + + def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): + super().__init__(client, operation, ['start_key', 'start_key_doc_id'], opts) + + def get_next(self) -> list[I]: + pass + + def _get_next_page_options(self, result: R): + pass + + def _items(self, result: R) -> list[I]: + return result.rows + + def _get_key(self, item: I) -> K: + return item.key + + def _get_id(self, item: I) -> str: + return item.id + + def _set_id(self, opts: dict, next_id: str): + opts['start_key_doc_id'] = next_id + +class _BookmarkPager(_BasePager): + + def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): + super().__init__(client, operation, ['bookmark'], opts) + + def _get_next_page_options(self, result: R) -> dict: + pass + + def _get_bookmark(self, result: R): + return result.bookmark + + def _set_bookmark(self, opts:dict, bookmark:str) -> str: + opts['bookmark'] = bookmark + +class _AllDocsBasePager(_KeyPager[str]): + + def _result_converter(self): + return AllDocsResult.from_dict + + def _set_id(self, opts: dict, next_id: str): + # no-op for AllDocs paging + pass + +class _AllDocsPager(_AllDocsBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_all_docs, opts) + +class _AllDocsPartitionPager(_AllDocsBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_partition_all_docs, opts) + +class _DesignDocsPager(_AllDocsBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_design_docs, opts) + +class _FindBasePager(_BookmarkPager): + + def _items(self, result: FindResult): + return result.docs + + def _result_converter(self): + return FindResult.from_dict + +class _FindPager(_FindBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_find, opts) + +class _FindPartitionPager(_FindBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_partition_find, opts) + +class _SearchBasePager(_BookmarkPager): + + def _items(self, result: SearchResult): + return result.rows + + def _result_converter(self): + return SearchResult.from_dict + +class _SearchPager(_SearchBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_search, opts) + +class _SearchPartitionPager(_SearchBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_partition_search, opts) + +class _ViewBasePager(_KeyPager[any]): + + def _result_converter(self): + return AllDocsResult.from_dict + +class _ViewPager(_ViewBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_view, opts) + +class _ViewPartitionPager(_ViewBasePager): + + def __init__(self, client: CloudantV1, opts: dict): + super().__init__(client, client.post_partition_view, opts) From dce16daa7bc9f1131de8ca2695951e526cd9e37d Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 7 Feb 2025 14:55:56 +0000 Subject: [PATCH 02/27] feat: base pager implementation --- ibmcloudant/features/pagination.py | 64 +++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 285afa2d..4e9abe08 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -24,6 +24,7 @@ from collections.abc import Callable from enum import auto, Enum from functools import partial +from types import MappingProxyType from typing import Generic, Protocol, TypeVar from ibm_cloud_sdk_core import DetailedResponse @@ -58,24 +59,26 @@ class Pager(Protocol[R, I]): the operation types in PagerType. """ + @abstractmethod def has_next(self) -> bool: """ - returns True if there may be another page of results, False otherwise + returns False if there are no more pages """ - pass + ... @abstractmethod def get_next(self) -> list[I]: """ returns the next page of results """ - pass + ... + @abstractmethod def get_all(self) -> list[I]: """ returns all the pages of results in single list """ - pass + ... @classmethod def new_pager(cls, client:CloudantV1, type: PagerType, **kwargs,): @@ -95,16 +98,53 @@ def __init__(self, page_opts: list[str], opts:dict): self._client = client - # TODO split the opts into fixed and page parts based on page_opts + self._has_next = True + # split the opts into fixed and page parts based on page_opts self._next_page_opts = {} fixed_opts = {} + fixed_opts |= opts + self._page_size = self.page_size_from_opts_limit(fixed_opts) + fixed_opts['limit'] = self._page_size + for k in page_opts: + if v := fixed_opts.pop(k, None): + self._next_page_opts[k] = v + fixed_opts = MappingProxyType(fixed_opts) # Partial method with the fixed ops self._next_request_function = partial(operation, **fixed_opts) - def _next_request(self) -> R: + def has_next(self) -> bool: + return self._has_next + + def get_next(self) -> list[I]: + if self.has_next(): + return self._next_request() + raise StopIteration() + + def get_all(self) -> list[I]: + all_items = [] + for page in self: + all_items.extend(page) + return all_items + + def __iter__(self): + return self + + def __next__(self): + return self.get_next() + + def _next_request(self) -> list[I]: response: DetailedResponse = self._next_request_function(**self._next_page_opts) result: dict = response.get_result() - return self._result_converter()(result) + typed_result: R = self._result_converter()(result) + items: list[I] = self._items(typed_result) + if len(items) < self._page_size: + self._has_next = False + else: + self._next_page_opts = self._get_next_page_options(typed_result) + return items + + def page_size_from_opts_limit(self, opts:dict) -> int: + return opts.get('limit', 20) @abstractmethod def _result_converter(self) -> Callable[[dict], R]: @@ -123,7 +163,7 @@ class _KeyPager(_BasePager, Generic[K]): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): super().__init__(client, operation, ['start_key', 'start_key_doc_id'], opts) - def get_next(self) -> list[I]: + def _next_request(self) -> list[I]: pass def _get_next_page_options(self, result: R): @@ -138,9 +178,15 @@ def _get_key(self, item: I) -> K: def _get_id(self, item: I) -> str: return item.id + def _set_key(self, opts: dict, next_key: K): + opts['start_key'] = next_key + def _set_id(self, opts: dict, next_id: str): opts['start_key_doc_id'] = next_id + def _page_size_from_opts_limit(self, opts:dict) -> int: + return super()._page_size_from_opts_limit(opts) + 1 + class _BookmarkPager(_BasePager): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): @@ -158,7 +204,7 @@ def _set_bookmark(self, opts:dict, bookmark:str) -> str: class _AllDocsBasePager(_KeyPager[str]): def _result_converter(self): - return AllDocsResult.from_dict + pass def _set_id(self, opts: dict, next_id: str): # no-op for AllDocs paging From 28172c56a0c65d51269380cdb81cc41c545a9306 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 26 Feb 2025 12:20:42 +0000 Subject: [PATCH 03/27] fix: make pages immutable --- ibmcloudant/features/pagination.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 4e9abe08..0c1790f7 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -67,14 +67,14 @@ def has_next(self) -> bool: ... @abstractmethod - def get_next(self) -> list[I]: + def get_next(self) -> tuple[I]: """ returns the next page of results """ ... @abstractmethod - def get_all(self) -> list[I]: + def get_all(self) -> tuple[I]: """ returns all the pages of results in single list """ @@ -115,16 +115,16 @@ def __init__(self, def has_next(self) -> bool: return self._has_next - def get_next(self) -> list[I]: + def get_next(self) -> tuple[I]: if self.has_next(): - return self._next_request() + return (*self._next_request(),) raise StopIteration() - def get_all(self) -> list[I]: + def get_all(self) -> tuple[I]: all_items = [] for page in self: all_items.extend(page) - return all_items + return (*all_items,) def __iter__(self): return self From 6c94dc2781a92dc2aee087dfd4dfb2ec82d8ada3 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 24 Feb 2025 17:40:57 +0000 Subject: [PATCH 04/27] feat: _KeyPager overrides --- ibmcloudant/features/pagination.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 0c1790f7..f743e71c 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -164,7 +164,10 @@ def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse super().__init__(client, operation, ['start_key', 'start_key_doc_id'], opts) def _next_request(self) -> list[I]: - pass + items: list[I] = super()._next_request() + if self.has_next(): + return items[:-1] + return items def _get_next_page_options(self, result: R): pass From 7f64e1fb9af800266b334eb00d2a84779ef7155d Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 25 Feb 2025 17:20:08 +0000 Subject: [PATCH 05/27] test: mock client update --- test/unit/features/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/features/conftest.py b/test/unit/features/conftest.py index d591e3de..e4e373e7 100644 --- a/test/unit/features/conftest.py +++ b/test/unit/features/conftest.py @@ -106,8 +106,7 @@ def limits(request): _BATCH_SIZE + 123, ] - -class ChangesFollowerBaseCase(unittest.TestCase): +class MockClientBaseCase(unittest.TestCase): @classmethod def setUpClass(cls): # Setup client env config @@ -117,6 +116,8 @@ def setUpClass(cls): service_name='TEST_SERVICE', ) +class ChangesFollowerBaseCase(MockClientBaseCase): + def prepare_mock_changes( self, batches=0, From b5cc8bc0d1266305df94208f4609e4c74cf9db71 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 25 Feb 2025 17:20:37 +0000 Subject: [PATCH 06/27] test: _BasePager tests --- test/unit/features/test_pagination_base.py | 315 +++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 test/unit/features/test_pagination_base.py diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py new file mode 100644 index 00000000..c8d61c77 --- /dev/null +++ b/test/unit/features/test_pagination_base.py @@ -0,0 +1,315 @@ +# coding: utf-8 + +# © Copyright IBM Corporation 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable +from itertools import batched +from ibm_cloud_sdk_core import DetailedResponse +from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow +from ibmcloudant.features.pagination import _BasePager, Pager +from conftest import MockClientBaseCase + +class TestPager(_BasePager): + """ + A test subclass of the _BasePager under test. + """ + def _result_converter(self) -> Callable[[dict], ViewResult]: + return lambda d: ViewResult.from_dict(d) + + def _items(self, result: ViewResult) -> tuple[ViewResultRow]: + return result.rows + + def _get_next_page_options(self, result: ViewResult) -> dict: + if len(result.rows ) == 0: + self.assertFail("Test failure: tried to setNextPageOptions on empty page.") + else: + return {'start_key': result.rows[-1].key} + +class MockPageReponses: + """ + Test class for mocking page responses. + """ + def __init__(self, total_items: int, page_size: int): + self.total_items: int = total_items + self.page_size: int = page_size + self.pages = self.generator() + self.expected_pages: list[list[ViewResultRow]] = [] + + def generator(self): + for page in batched(range(0, self.total_items), self.page_size): + rows = [{'id':str(i), 'key':i, 'value':i} for i in page] + yield DetailedResponse(response={'rows': rows}) + yield DetailedResponse(response={'rows': []}) + + def get_next_page(self, **kwargs): + # ignore kwargs + # get next page + page = next(self.pages) + # convert to an expected page + self.expected_pages.append(ViewResult.from_dict(page.get_result()).rows) + return page + + def get_expected_page(self, page: int) -> list[ViewResultRow]: + return self.expected_pages[page - 1] + + def all_expected_items(self) -> list[ViewResultRow]: + all_items: list[ViewResultRow] = [] + for page in self.expected_pages: + all_items.extend(page) + return all_items + +class TestBasePager(MockClientBaseCase): + def test_init(self): + operation = self.client.post_view + opts = {'db': 'test', 'limit': 20} + pager: Pager = TestPager(self.client, operation, [], opts) + # Assert client is set + self.assertEqual(pager._client, self.client, 'The supplied client should be set.') + # Assert operation is set + self.assertIsNotNone(pager._next_request_function, 'The operation function should be set.') + # Assert partial function parts are as expected + self.assertEqual(pager._next_request_function.func, operation, 'The partial function should be the operation.') + self.assertEqual(pager._next_request_function.keywords, opts, 'The partial function kwargs should be the options.') + + def test_partial_options(self): + operation = self.client.post_view + static_opts = {'db': 'test', 'limit': 20, 'baz': 'faz'} + page_opts = {'foo': 'boo', 'bar': 'far'} + opts = {**static_opts, **page_opts} + # Use page_opts.keys() to pass the list of names for page options + pager: Pager = TestPager(self.client, operation, page_opts.keys(), opts) + # Assert partial function has only static opts + self.assertEqual(pager._next_request_function.keywords, static_opts, 'The partial function kwargs should be only the static options.') + # Assert next page options + self.assertEqual(pager._next_page_opts, page_opts, 'The next page options should match the expected.') + + def test_default_page_size(self): + operation = self.client.post_view + opts = {'db': 'test'} + pager: Pager = TestPager(self.client, operation, [], opts) + # Assert the default page size + expected_page_size = 20 + self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') + self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') + + def test_limit_page_size(self): + operation = self.client.post_view + opts = {'db': 'test', 'limit': 42} + pager: Pager = TestPager(self.client, operation, [], opts) + # Assert the provided page size + expected_page_size = 42 + self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') + self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') + + def test_has_next_initially_true(self): + operation = self.client.post_view + opts = {'limit': 1} + pager: Pager = TestPager(self.client, operation, [], opts) + # Assert has_next() + self.assertTrue(pager.has_next(), 'has_next() should initially return True.') + + def test_has_next_true_for_result_equal_to_limit(self): + page_size = 1 + # Init with mock that returns only a single row + pager: Pager = TestPager( + self.client, + MockPageReponses(1, page_size).get_next_page, + [], + {'limit': page_size}) + # Get first page with 1 result + pager.get_next() + # Assert has_next() + self.assertTrue(pager.has_next(), 'has_next() should return True.') + + def test_has_next_false_for_result_less_than_limit(self): + page_size = 1 + # Init with mock that returns zero rows + pager: Pager = TestPager( + self.client, + MockPageReponses(0, page_size).get_next_page, + [], + {'limit': page_size}) + # Get first page with 0 result + pager.get_next() + # Assert has_next() + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + def test_get_next_first_page(self): + page_size = 25 + # Mock that returns one page of 25 items + mock = MockPageReponses(page_size, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + # Get first page + actual_page: list[ViewResultRow] = pager.get_next() + # Assert first page + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + + def test_get_next_two_pages(self): + page_size = 3 + # Mock that returns two pages of 3 items + mock = MockPageReponses(2*page_size, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + # Get first page + actual_page_1: list[ViewResultRow] = pager.get_next() + # Assert first page + self.assertSequenceEqual(actual_page_1, mock.get_expected_page(1), "The actual page should match the expected page") + # Assert has_next + self.assertTrue(pager.has_next(), 'has_next() should return True.') + # Get second page + actual_page_2: list[ViewResultRow] = pager.get_next() + # Assert first page + self.assertSequenceEqual(actual_page_2, mock.get_expected_page(2), "The actual page should match the expected page") + # Assert has_next, True since page is not smaller than limit + self.assertTrue(pager.has_next(), 'has_next() should return True.') + + def test_get_next_until_empty(self): + page_size = 3 + # Mock that returns 3 pages of 3 items + mock = MockPageReponses(3*page_size, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + page_count = 0 + actual_items = [] + while pager.has_next(): + page_count += 1 + page = pager.get_next() + # Assert each page is the same or smaller than the limit to confirm all results not in one page + self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") + actual_items.extend(page) + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") + + def test_get_next_until_smaller(self): + page_size = 3 + # Mock that returns 3 pages of 3 items, then 1 more page with 1 item + mock = MockPageReponses(3*page_size + 1, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + page_count = 0 + actual_items = [] + while pager.has_next(): + page_count += 1 + page = pager.get_next() + # Assert each page is the same or smaller than the limit to confirm all results not in one page + self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") + actual_items.extend(page) + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") + + def test_get_next_exception(self): + page_size = 2 + # Mock that returns one page of one item + mock = MockPageReponses(page_size - 1, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + # Get first and only page + actual_page: list[ViewResultRow] = pager.get_next() + # Assert page + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + # Assert has_next() now False + self.assertFalse(pager.has_next(), 'has_next() should return False.') + # Assert StopIteraton on get_next() + with self.assertRaises(StopIteration): + pager.get_next() + + def test_get_all(self): + page_size = 11 + # Mock that returns 6 pages of 11 items, then 1 more page with 5 items + mock = MockPageReponses(71, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + actual_items = pager.get_all() + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + + def test_iter_next_first_page(self): + page_size = 7 + # Mock that returns two pages of 7 items + mock = MockPageReponses(2*page_size, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + # Get first page + actual_page: list[ViewResultRow] = next(pager) + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + + def test_iter(self): + page_size = 23 + mock = MockPageReponses(3*page_size-1, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + # Check pager is an iterator + page_number = 0 + for page in pager: + page_number += 1 + self.assertSequenceEqual(page, mock.get_expected_page(page_number), "The actual page should match the expected page") + # Asser the correct number of pages + self.assertEqual(page_number, 3, 'There should have been 3 pages.') + + def test_pages_immutable(self): + page_size = 1 + mock = MockPageReponses(page_size, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + # Get page + actual_page: list[ViewResultRow] = pager.get_next() + # Assert immutable tuple type + self.assertIsInstance(actual_page, tuple) + + def test_set_next_page_options(self): + page_size = 1 + mock = MockPageReponses(5*page_size, page_size) + pager: Pager = TestPager( + self.client, + mock.get_next_page, + [], + {'limit': page_size}) + self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") + # Since we use a page size of 1, each next page options key, is the same as the element from the page and the page count + page_count = 0 + while pager.has_next(): + page = pager.get_next() + if pager.has_next(): + self.assertEqual(page_count, pager._next_page_opts.get('start_key'), "The key should increment per page.") + else: + self.assertEqual(page_count - 1, pager._next_page_opts.get('start_key'), "The options should not be set for the final page.") + page_count += 1 From 152f6cedaa75b978b66439fcbd10481768f9b435 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 17 Mar 2025 12:00:26 +0000 Subject: [PATCH 07/27] fix: clean up --- ibmcloudant/features/pagination.py | 31 +++++++++++----------- test/unit/features/test_pagination_base.py | 26 +++++++++--------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index f743e71c..352636fc 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -64,21 +64,21 @@ def has_next(self) -> bool: """ returns False if there are no more pages """ - ... + raise NotImplementedError() @abstractmethod def get_next(self) -> tuple[I]: """ returns the next page of results """ - ... + raise NotImplementedError() @abstractmethod def get_all(self) -> tuple[I]: """ returns all the pages of results in single list """ - ... + raise NotImplementedError() @classmethod def new_pager(cls, client:CloudantV1, type: PagerType, **kwargs,): @@ -88,7 +88,7 @@ def new_pager(cls, client:CloudantV1, type: PagerType, **kwargs,): type: PagerType - the operation type to paginate kwargs: dict - the options for the operation """ - pass + raise NotImplementedError() class _BasePager(Pager): @@ -97,20 +97,20 @@ def __init__(self, operation: Callable[..., DetailedResponse], page_opts: list[str], opts:dict): - self._client = client - self._has_next = True + self._client: CloudantV1 = client + self._has_next: bool = True # split the opts into fixed and page parts based on page_opts - self._next_page_opts = {} - fixed_opts = {} + self._next_page_opts: dict = {} + fixed_opts: dict = {} fixed_opts |= opts - self._page_size = self.page_size_from_opts_limit(fixed_opts) + self._page_size: int = self.page_size_from_opts_limit(fixed_opts) fixed_opts['limit'] = self._page_size for k in page_opts: if v := fixed_opts.pop(k, None): self._next_page_opts[k] = v fixed_opts = MappingProxyType(fixed_opts) # Partial method with the fixed ops - self._next_request_function = partial(operation, **fixed_opts) + self._next_request_function: function = partial(operation, **fixed_opts) def has_next(self) -> bool: return self._has_next @@ -148,15 +148,15 @@ def page_size_from_opts_limit(self, opts:dict) -> int: @abstractmethod def _result_converter(self) -> Callable[[dict], R]: - ... + raise NotImplementedError() @abstractmethod def _items(self, result: R) -> list[I]: - ... + raise NotImplementedError() @abstractmethod def _get_next_page_options(self, result: R) -> dict: - ... + raise NotImplementedError() class _KeyPager(_BasePager, Generic[K]): @@ -169,7 +169,7 @@ def _next_request(self) -> list[I]: return items[:-1] return items - def _get_next_page_options(self, result: R): + def _get_next_page_options(self, result: R) -> dict: pass def _items(self, result: R) -> list[I]: @@ -206,8 +206,9 @@ def _set_bookmark(self, opts:dict, bookmark:str) -> str: class _AllDocsBasePager(_KeyPager[str]): + @abstractmethod def _result_converter(self): - pass + raise NotImplementedError() def _set_id(self, opts: dict, next_id: str): # no-op for AllDocs paging diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py index c8d61c77..066859ea 100644 --- a/test/unit/features/test_pagination_base.py +++ b/test/unit/features/test_pagination_base.py @@ -37,7 +37,7 @@ def _get_next_page_options(self, result: ViewResult) -> dict: else: return {'start_key': result.rows[-1].key} -class MockPageReponses: +class MockPageResponses: """ Test class for mocking page responses. """ @@ -125,7 +125,7 @@ def test_has_next_true_for_result_equal_to_limit(self): # Init with mock that returns only a single row pager: Pager = TestPager( self.client, - MockPageReponses(1, page_size).get_next_page, + MockPageResponses(1, page_size).get_next_page, [], {'limit': page_size}) # Get first page with 1 result @@ -138,7 +138,7 @@ def test_has_next_false_for_result_less_than_limit(self): # Init with mock that returns zero rows pager: Pager = TestPager( self.client, - MockPageReponses(0, page_size).get_next_page, + MockPageResponses(0, page_size).get_next_page, [], {'limit': page_size}) # Get first page with 0 result @@ -149,7 +149,7 @@ def test_has_next_false_for_result_less_than_limit(self): def test_get_next_first_page(self): page_size = 25 # Mock that returns one page of 25 items - mock = MockPageReponses(page_size, page_size) + mock = MockPageResponses(page_size, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -163,7 +163,7 @@ def test_get_next_first_page(self): def test_get_next_two_pages(self): page_size = 3 # Mock that returns two pages of 3 items - mock = MockPageReponses(2*page_size, page_size) + mock = MockPageResponses(2*page_size, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -185,7 +185,7 @@ def test_get_next_two_pages(self): def test_get_next_until_empty(self): page_size = 3 # Mock that returns 3 pages of 3 items - mock = MockPageReponses(3*page_size, page_size) + mock = MockPageResponses(3*page_size, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -205,7 +205,7 @@ def test_get_next_until_empty(self): def test_get_next_until_smaller(self): page_size = 3 # Mock that returns 3 pages of 3 items, then 1 more page with 1 item - mock = MockPageReponses(3*page_size + 1, page_size) + mock = MockPageResponses(3*page_size + 1, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -225,7 +225,7 @@ def test_get_next_until_smaller(self): def test_get_next_exception(self): page_size = 2 # Mock that returns one page of one item - mock = MockPageReponses(page_size - 1, page_size) + mock = MockPageResponses(page_size - 1, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -244,7 +244,7 @@ def test_get_next_exception(self): def test_get_all(self): page_size = 11 # Mock that returns 6 pages of 11 items, then 1 more page with 5 items - mock = MockPageReponses(71, page_size) + mock = MockPageResponses(71, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -256,7 +256,7 @@ def test_get_all(self): def test_iter_next_first_page(self): page_size = 7 # Mock that returns two pages of 7 items - mock = MockPageReponses(2*page_size, page_size) + mock = MockPageResponses(2*page_size, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -268,7 +268,7 @@ def test_iter_next_first_page(self): def test_iter(self): page_size = 23 - mock = MockPageReponses(3*page_size-1, page_size) + mock = MockPageResponses(3*page_size-1, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -284,7 +284,7 @@ def test_iter(self): def test_pages_immutable(self): page_size = 1 - mock = MockPageReponses(page_size, page_size) + mock = MockPageResponses(page_size, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, @@ -297,7 +297,7 @@ def test_pages_immutable(self): def test_set_next_page_options(self): page_size = 1 - mock = MockPageReponses(5*page_size, page_size) + mock = MockPageResponses(5*page_size, page_size) pager: Pager = TestPager( self.client, mock.get_next_page, From c804fac5a212ccc5993b84ee5e4236e5f61e62fe Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 17 Mar 2025 12:02:09 +0000 Subject: [PATCH 08/27] feat: update default page size to 200 --- ibmcloudant/features/pagination.py | 2 +- test/unit/features/test_pagination_base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 352636fc..f105f42f 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -144,7 +144,7 @@ def _next_request(self) -> list[I]: return items def page_size_from_opts_limit(self, opts:dict) -> int: - return opts.get('limit', 20) + return opts.get('limit', 200) @abstractmethod def _result_converter(self) -> Callable[[dict], R]: diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py index 066859ea..ab9735bc 100644 --- a/test/unit/features/test_pagination_base.py +++ b/test/unit/features/test_pagination_base.py @@ -100,7 +100,7 @@ def test_default_page_size(self): opts = {'db': 'test'} pager: Pager = TestPager(self.client, operation, [], opts) # Assert the default page size - expected_page_size = 20 + expected_page_size = 200 self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') From e42371b1374e9929362c5375c4d3e09d897026cb Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 18 Mar 2025 15:28:12 +0000 Subject: [PATCH 09/27] feat: reusable get_all --- ibmcloudant/features/pagination.py | 24 +- test/unit/features/test_pagination_base.py | 330 ++++++++++++--------- 2 files changed, 198 insertions(+), 156 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index f105f42f..8c19e7c9 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -96,21 +96,30 @@ def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], page_opts: list[str], - opts:dict): + opts: dict): self._client: CloudantV1 = client self._has_next: bool = True # split the opts into fixed and page parts based on page_opts self._next_page_opts: dict = {} - fixed_opts: dict = {} - fixed_opts |= opts + fixed_opts: dict = dict(opts) + # Get the page size and set the limit acoordingly self._page_size: int = self.page_size_from_opts_limit(fixed_opts) fixed_opts['limit'] = self._page_size + # Remove the options that change per page for k in page_opts: if v := fixed_opts.pop(k, None): self._next_page_opts[k] = v + self._initial_opts = dict(opts) fixed_opts = MappingProxyType(fixed_opts) # Partial method with the fixed ops - self._next_request_function: function = partial(operation, **fixed_opts) + self._next_request_function: Callable[..., DetailedResponse] = partial(operation, **fixed_opts) + + def _new_copy(self) -> Pager: + # Make and return new instance of the specific sub-class in use + return type(self)( + self._client, + { **self._next_request_function.keywords, **self._initial_opts }) + def has_next(self) -> bool: return self._has_next @@ -127,7 +136,7 @@ def get_all(self) -> tuple[I]: return (*all_items,) def __iter__(self): - return self + return self._new_copy() def __next__(self): return self.get_next() @@ -206,9 +215,8 @@ def _set_bookmark(self, opts:dict, bookmark:str) -> str: class _AllDocsBasePager(_KeyPager[str]): - @abstractmethod - def _result_converter(self): - raise NotImplementedError() + def _result_converter(self) -> Callable[[dict], AllDocsResult]: + return AllDocsResult.from_dict def _set_id(self, opts: dict, next_id: str): # no-op for AllDocs paging diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py index ab9735bc..467571e5 100644 --- a/test/unit/features/test_pagination_base.py +++ b/test/unit/features/test_pagination_base.py @@ -16,6 +16,7 @@ from collections.abc import Callable from itertools import batched +from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow from ibmcloudant.features.pagination import _BasePager, Pager @@ -25,6 +26,12 @@ class TestPager(_BasePager): """ A test subclass of the _BasePager under test. """ + operation: Callable = None + page_keys: list[str] = [] + + def __init__(self, client, opts): + super().__init__(client, TestPager.operation or client.post_view, TestPager.page_keys, opts) + def _result_converter(self) -> Callable[[dict], ViewResult]: return lambda d: ViewResult.from_dict(d) @@ -74,7 +81,7 @@ class TestBasePager(MockClientBaseCase): def test_init(self): operation = self.client.post_view opts = {'db': 'test', 'limit': 20} - pager: Pager = TestPager(self.client, operation, [], opts) + pager: Pager = TestPager(self.client, opts) # Assert client is set self.assertEqual(pager._client, self.client, 'The supplied client should be set.') # Assert operation is set @@ -84,232 +91,259 @@ def test_init(self): self.assertEqual(pager._next_request_function.keywords, opts, 'The partial function kwargs should be the options.') def test_partial_options(self): - operation = self.client.post_view static_opts = {'db': 'test', 'limit': 20, 'baz': 'faz'} page_opts = {'foo': 'boo', 'bar': 'far'} opts = {**static_opts, **page_opts} # Use page_opts.keys() to pass the list of names for page options - pager: Pager = TestPager(self.client, operation, page_opts.keys(), opts) + with patch('test_pagination_base.TestPager.page_keys', page_opts.keys()): + pager: Pager = TestPager(self.client, opts) # Assert partial function has only static opts self.assertEqual(pager._next_request_function.keywords, static_opts, 'The partial function kwargs should be only the static options.') # Assert next page options self.assertEqual(pager._next_page_opts, page_opts, 'The next page options should match the expected.') def test_default_page_size(self): - operation = self.client.post_view opts = {'db': 'test'} - pager: Pager = TestPager(self.client, operation, [], opts) + pager: Pager = TestPager(self.client, opts) # Assert the default page size expected_page_size = 200 self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') def test_limit_page_size(self): - operation = self.client.post_view opts = {'db': 'test', 'limit': 42} - pager: Pager = TestPager(self.client, operation, [], opts) + pager: Pager = TestPager(self.client, opts) # Assert the provided page size expected_page_size = 42 self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') def test_has_next_initially_true(self): - operation = self.client.post_view opts = {'limit': 1} - pager: Pager = TestPager(self.client, operation, [], opts) + pager: Pager = TestPager(self.client, opts) # Assert has_next() self.assertTrue(pager.has_next(), 'has_next() should initially return True.') def test_has_next_true_for_result_equal_to_limit(self): page_size = 1 # Init with mock that returns only a single row - pager: Pager = TestPager( - self.client, - MockPageResponses(1, page_size).get_next_page, - [], - {'limit': page_size}) - # Get first page with 1 result - pager.get_next() - # Assert has_next() - self.assertTrue(pager.has_next(), 'has_next() should return True.') + with patch('test_pagination_base.TestPager.operation', MockPageResponses(1, page_size).get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get first page with 1 result + pager.get_next() + # Assert has_next() + self.assertTrue(pager.has_next(), 'has_next() should return True.') def test_has_next_false_for_result_less_than_limit(self): page_size = 1 # Init with mock that returns zero rows - pager: Pager = TestPager( - self.client, - MockPageResponses(0, page_size).get_next_page, - [], - {'limit': page_size}) - # Get first page with 0 result - pager.get_next() - # Assert has_next() - self.assertFalse(pager.has_next(), 'has_next() should return False.') + with patch('test_pagination_base.TestPager.operation', MockPageResponses(0, page_size).get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get first page with 0 result + pager.get_next() + # Assert has_next() + self.assertFalse(pager.has_next(), 'has_next() should return False.') def test_get_next_first_page(self): page_size = 25 # Mock that returns one page of 25 items mock = MockPageResponses(page_size, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - # Get first page - actual_page: list[ViewResultRow] = pager.get_next() - # Assert first page - self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get first page + actual_page: list[ViewResultRow] = pager.get_next() + # Assert first page + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") def test_get_next_two_pages(self): page_size = 3 # Mock that returns two pages of 3 items mock = MockPageResponses(2*page_size, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - # Get first page - actual_page_1: list[ViewResultRow] = pager.get_next() - # Assert first page - self.assertSequenceEqual(actual_page_1, mock.get_expected_page(1), "The actual page should match the expected page") - # Assert has_next - self.assertTrue(pager.has_next(), 'has_next() should return True.') - # Get second page - actual_page_2: list[ViewResultRow] = pager.get_next() - # Assert first page - self.assertSequenceEqual(actual_page_2, mock.get_expected_page(2), "The actual page should match the expected page") - # Assert has_next, True since page is not smaller than limit - self.assertTrue(pager.has_next(), 'has_next() should return True.') + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get first page + actual_page_1: list[ViewResultRow] = pager.get_next() + # Assert first page + self.assertSequenceEqual(actual_page_1, mock.get_expected_page(1), "The actual page should match the expected page") + # Assert has_next + self.assertTrue(pager.has_next(), 'has_next() should return True.') + # Get second page + actual_page_2: list[ViewResultRow] = pager.get_next() + # Assert first page + self.assertSequenceEqual(actual_page_2, mock.get_expected_page(2), "The actual page should match the expected page") + # Assert has_next, True since page is not smaller than limit + self.assertTrue(pager.has_next(), 'has_next() should return True.') def test_get_next_until_empty(self): page_size = 3 # Mock that returns 3 pages of 3 items mock = MockPageResponses(3*page_size, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - page_count = 0 - actual_items = [] - while pager.has_next(): - page_count += 1 - page = pager.get_next() - # Assert each page is the same or smaller than the limit to confirm all results not in one page - self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") - actual_items.extend(page) - self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") - self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + page_count = 0 + actual_items = [] + while pager.has_next(): + page_count += 1 + page = pager.get_next() + # Assert each page is the same or smaller than the limit to confirm all results not in one page + self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") + actual_items.extend(page) + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") def test_get_next_until_smaller(self): page_size = 3 # Mock that returns 3 pages of 3 items, then 1 more page with 1 item mock = MockPageResponses(3*page_size + 1, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - page_count = 0 - actual_items = [] - while pager.has_next(): - page_count += 1 - page = pager.get_next() - # Assert each page is the same or smaller than the limit to confirm all results not in one page - self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") - actual_items.extend(page) - self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") - self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + page_count = 0 + actual_items = [] + while pager.has_next(): + page_count += 1 + page = pager.get_next() + # Assert each page is the same or smaller than the limit to confirm all results not in one page + self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") + actual_items.extend(page) + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") def test_get_next_exception(self): page_size = 2 # Mock that returns one page of one item mock = MockPageResponses(page_size - 1, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - # Get first and only page - actual_page: list[ViewResultRow] = pager.get_next() - # Assert page - self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") - # Assert has_next() now False - self.assertFalse(pager.has_next(), 'has_next() should return False.') - # Assert StopIteraton on get_next() - with self.assertRaises(StopIteration): - pager.get_next() + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get first and only page + actual_page: list[ViewResultRow] = pager.get_next() + # Assert page + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + # Assert has_next() now False + self.assertFalse(pager.has_next(), 'has_next() should return False.') + # Assert StopIteraton on get_next() + with self.assertRaises(StopIteration): + pager.get_next() def test_get_all(self): page_size = 11 # Mock that returns 6 pages of 11 items, then 1 more page with 5 items mock = MockPageResponses(71, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - actual_items = pager.get_all() - self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + actual_items = pager.get_all() + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") def test_iter_next_first_page(self): page_size = 7 # Mock that returns two pages of 7 items mock = MockPageResponses(2*page_size, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - # Get first page - actual_page: list[ViewResultRow] = next(pager) - self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get first page + actual_page: list[ViewResultRow] = next(pager) + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") def test_iter(self): page_size = 23 mock = MockPageResponses(3*page_size-1, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - # Check pager is an iterator - page_number = 0 - for page in pager: - page_number += 1 - self.assertSequenceEqual(page, mock.get_expected_page(page_number), "The actual page should match the expected page") - # Asser the correct number of pages - self.assertEqual(page_number, 3, 'There should have been 3 pages.') + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Check pager is an iterator + page_number = 0 + for page in pager: + page_number += 1 + self.assertSequenceEqual(page, mock.get_expected_page(page_number), "The actual page should match the expected page") + # Asser the correct number of pages + self.assertEqual(page_number, 3, 'There should have been 3 pages.') def test_pages_immutable(self): page_size = 1 mock = MockPageResponses(page_size, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - # Get page - actual_page: list[ViewResultRow] = pager.get_next() - # Assert immutable tuple type - self.assertIsInstance(actual_page, tuple) + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + # Get page + actual_page: list[ViewResultRow] = pager.get_next() + # Assert immutable tuple type + self.assertIsInstance(actual_page, tuple) def test_set_next_page_options(self): page_size = 1 mock = MockPageResponses(5*page_size, page_size) - pager: Pager = TestPager( - self.client, - mock.get_next_page, - [], - {'limit': page_size}) - self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") - # Since we use a page size of 1, each next page options key, is the same as the element from the page and the page count - page_count = 0 - while pager.has_next(): - page = pager.get_next() - if pager.has_next(): - self.assertEqual(page_count, pager._next_page_opts.get('start_key'), "The key should increment per page.") - else: - self.assertEqual(page_count - 1, pager._next_page_opts.get('start_key'), "The options should not be set for the final page.") - page_count += 1 + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") + # Since we use a page size of 1, each next page options key, is the same as the element from the page and the page count + page_count = 0 + while pager.has_next(): + page = pager.get_next() + if pager.has_next(): + self.assertEqual(page_count, pager._next_page_opts.get('start_key'), "The key should increment per page.") + else: + self.assertEqual(page_count - 1, pager._next_page_opts.get('start_key'), "The options should not be set for the final page.") + page_count += 1 + + def test_get_next_resumes_after_error(self): + page_size = 1 + mock = MockPageResponses(3*page_size, page_size) + with patch('test_pagination_base.TestPager.operation', mock.get_next_page): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") + self.assertEqual(0, pager._next_page_opts.get('start_key'), "The start_key should be 0 for the second page.") + with patch('ibmcloudant.features.pagination._BasePager._next_request', Exception('test exception')): + with self.assertRaises(Exception): + pager.get_next() + self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertEqual(0, pager._next_page_opts.get('start_key'), "The start_key should still be 0 for the second page.") + second_page = pager.get_next() + self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") + self.assertTrue(pager.has_next(), 'has_next() should return False.') + + def test_get_all_restarts_after_error(self): + page_size = 1 + mock = MockPageResponses(2*page_size, page_size) + first_page = mock.get_next_page() + # mock response order + # first page, error, first page replay, second page + mockmock = Mock(side_effect=[ + first_page, + Exception('test exception'), + first_page, + mock.get_next_page() + ]) + with patch('test_pagination_base.TestPager.operation', mockmock): + pager: Pager = TestPager( + self.client, + {'limit': page_size}) + with self.assertRaises(Exception): + pager.get_all() + # After the error we should not have a start_key set + self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should be None.") + self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), "The results should match all the pages.") From b3518130478a39a9a5ebaa1a21403f764d182834 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 17 Mar 2025 12:01:13 +0000 Subject: [PATCH 10/27] feat: _KeyPager --- ibmcloudant/features/pagination.py | 55 ++++-- test/unit/features/test_pagination_key.py | 214 ++++++++++++++++++++++ 2 files changed, 249 insertions(+), 20 deletions(-) create mode 100644 test/unit/features/test_pagination_key.py diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 8c19e7c9..3f0c6419 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -103,7 +103,7 @@ def __init__(self, self._next_page_opts: dict = {} fixed_opts: dict = dict(opts) # Get the page size and set the limit acoordingly - self._page_size: int = self.page_size_from_opts_limit(fixed_opts) + self._page_size: int = self._page_size_from_opts_limit(fixed_opts) fixed_opts['limit'] = self._page_size # Remove the options that change per page for k in page_opts: @@ -152,7 +152,7 @@ def _next_request(self) -> list[I]: self._next_page_opts = self._get_next_page_options(typed_result) return items - def page_size_from_opts_limit(self, opts:dict) -> int: + def _page_size_from_opts_limit(self, opts:dict) -> int: return opts.get('limit', 200) @abstractmethod @@ -171,34 +171,38 @@ class _KeyPager(_BasePager, Generic[K]): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): super().__init__(client, operation, ['start_key', 'start_key_doc_id'], opts) + self._boundary_failure: str | None = None def _next_request(self) -> list[I]: + if self._boundary_failure is not None: + raise Exception(self._boundary_failure) items: list[I] = super()._next_request() if self.has_next(): - return items[:-1] + last_item: I = items.pop() + if len(items) > 0: + # Get, but don't remove the last item from the list + penultimate_item: I = items[-1] + self._boundary_failure: str | None = self.check_boundary(penultimate_item, last_item) return items def _get_next_page_options(self, result: R) -> dict: - pass + # last item is used for next page options + last_item = self._items(result)[-1] + return { + 'start_key': last_item.key, + 'start_key_doc_id': last_item.id, + } def _items(self, result: R) -> list[I]: return result.rows - def _get_key(self, item: I) -> K: - return item.key - - def _get_id(self, item: I) -> str: - return item.id - - def _set_key(self, opts: dict, next_key: K): - opts['start_key'] = next_key - - def _set_id(self, opts: dict, next_id: str): - opts['start_key_doc_id'] = next_id - def _page_size_from_opts_limit(self, opts:dict) -> int: return super()._page_size_from_opts_limit(opts) + 1 + @abstractmethod + def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: + raise NotImplementedError() + class _BookmarkPager(_BasePager): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): @@ -218,9 +222,14 @@ class _AllDocsBasePager(_KeyPager[str]): def _result_converter(self) -> Callable[[dict], AllDocsResult]: return AllDocsResult.from_dict - def _set_id(self, opts: dict, next_id: str): - # no-op for AllDocs paging - pass + def _get_next_page_options(self, result: R) -> dict: + # Remove start_key_doc_id for all_docs paging + opts: dict = super()._get_next_page_options(result) + del opts.start_key_doc_id + + def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: + # IDs are always unique in _all_docs pagers so return None + return None class _AllDocsPager(_AllDocsBasePager): @@ -276,7 +285,13 @@ def __init__(self, client: CloudantV1, opts: dict): class _ViewBasePager(_KeyPager[any]): def _result_converter(self): - return AllDocsResult.from_dict + return ViewResult.from_dict + + def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: + if penultimate_item.id == (boundary_id:= last_item.id) \ + and penultimate_item.key == (boundary_key:= last_item.key): + return f'Cannot paginate on a boundary containing identical keys {boundary_key} and document IDs {boundary_id}' + return None class _ViewPager(_ViewBasePager): diff --git a/test/unit/features/test_pagination_key.py b/test/unit/features/test_pagination_key.py new file mode 100644 index 00000000..e081877b --- /dev/null +++ b/test/unit/features/test_pagination_key.py @@ -0,0 +1,214 @@ +# coding: utf-8 + +# © Copyright IBM Corporation 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable +from itertools import batched +from unittest.mock import Mock, patch +from ibm_cloud_sdk_core import DetailedResponse +from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow +from ibmcloudant.features.pagination import _KeyPager, Pager +from conftest import MockClientBaseCase + +class KeyTestPager(_KeyPager): + """ + A test subclass of the _KeyPager under test. + """ + operation: Callable = None + boundary_func: Callable = lambda p,l: None + + def __init__(self, client, opts): + super().__init__(client, KeyTestPager.operation or client.post_view, opts) + + def _result_converter(self) -> Callable[[dict], ViewResult]: + return lambda d: ViewResult.from_dict(d) + + def _items(self, result: ViewResult) -> tuple[ViewResultRow]: + return result.rows + + def _get_next_page_options(self, result: ViewResult) -> dict: + if len(result.rows ) == 0: + self.assertFail("Test failure: tried to setNextPageOptions on empty page.") + else: + return {'start_key': result.rows[-1].key} + + def check_boundary(self, penultimate_item, last_item): + return KeyTestPager.boundary_func(penultimate_item, last_item) + +class MockPageResponses: + """ + Test class for mocking page responses. + """ + def __init__(self, total_items: int, page_size: int): + self.total_items: int = total_items + self.page_size: int = page_size + self.pages = self.generator() + self.expected_pages: list[list[ViewResultRow]] = [] + + def generator(self): + for page in batched(range(0, self.total_items), self.page_size): + rows = [{'id':str(i), 'key':i, 'value':i} for i in page] + # Add an n+1 row for key based paging if more pages + if (n_plus_one := page[-1] + 1) < self.total_items: + rows.append({'id':str(n_plus_one), 'key':n_plus_one, 'value':n_plus_one}) + yield DetailedResponse(response={'rows': rows}) + yield DetailedResponse(response={'rows': []}) + + def get_next_page(self, **kwargs): + # ignore kwargs + # get next page + page = next(self.pages) + # convert to an expected page, removing the n+1 row if needed + result = ViewResult.from_dict(page.get_result()) + if len(result.rows) > self.page_size: + self.expected_pages.append(result.rows[:-1]) + else: + self.expected_pages.append(result.rows) + return page + + def get_expected_page(self, page: int) -> list[ViewResultRow]: + return self.expected_pages[page - 1] + + def all_expected_items(self) -> list[ViewResultRow]: + all_items: list[ViewResultRow] = [] + for page in self.expected_pages: + all_items.extend(page) + return all_items + +class TestKeyPager(MockClientBaseCase): + + # Test page size default (+1) + def test_default_page_size(self): + pager: Pager = KeyTestPager(self.client, {}) + # Assert the limit default as page size + self.assertEqual(pager._page_size, 201, 'The page size should be one more than the default limit.') + + # Test page size limit (+1) + def test_limit_page_size(self): + pager: Pager = KeyTestPager(self.client, {'limit': 42}) + # Assert the limit provided as page size + self.assertEqual(pager._page_size, 43, 'The page size should be one more than the default limit.') + + # Test all items on page when no more pages + def test_get_next_page_less_than_limit(self): + page_size = 21 + mock = MockPageResponses(page_size, page_size) + with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): + pager = KeyTestPager(self.client, {'limit': page_size}) + # Get and assert first page + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page') + # Assert page size + self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') + # Assert has_next False because n+1 limit is 1 more than user page size + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + # Test correct items on page when n+1 + def test_get_next_page_equal_to_limit(self): + page_size = 14 + mock = MockPageResponses(page_size+1, page_size) + with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): + pager = KeyTestPager(self.client, {'limit': page_size}) + # Get and assert first page + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') + # Assert page size + self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') + # Assert has_next True + self.assertTrue(pager.has_next(), 'has_next() should return True.') + # Get and assert second page + second_page = pager.get_next() + self.assertEqual(len(second_page), 1 , 'The second page should have one item.') + # Note row keys are zero indexed so n+1 element that is first item on second page matches page size + self.assertEqual(second_page[0].key, page_size, 'The first item key on the second page should match the page size number.') + self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + # Test correct items on page when n+more + def test_get_next_page_greater_than_limit(self): + page_size = 7 + mock = MockPageResponses(page_size+2, page_size) + with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): + pager = KeyTestPager(self.client, {'limit': page_size}) + # Get and assert first page + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') + # Assert page size + self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') + # Assert has_next True + self.assertTrue(pager.has_next(), 'has_next() should return True.') + # Get and assert second page + second_page = pager.get_next() + self.assertEqual(len(second_page), 2 , 'The second page should have two items.') + # Note row keys are zero indexed so n+1 element that is first item on second page matches page size + self.assertEqual(second_page[0].key, page_size, 'The first item key on the second page should match the page size number.') + self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + # Test getting all items + def test_get_all(self): + page_size = 3 + mock = MockPageResponses(page_size*12, page_size) + with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): + pager = KeyTestPager(self.client, {'limit': page_size}) + # Get and assert all items + self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), 'The results should match all the pages.') + + def test_no_boundary_check_by_default(self): + mock_rows = [ + {'id': '1', 'key': 1, 'value': 1}, + {'id': '1', 'key': 1, 'value': 1} + ] + expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows + mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) + with patch('test_pagination_key.KeyTestPager.operation', mockmock): + pager = KeyTestPager(self.client, {'limit': 1}) + # Get and assert page + self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) + + def test_boundary_failure_throws_on_get_next(self): + mock_rows = [ + {'id': '1', 'key': 1, 'value': 1}, + {'id': '1', 'key': 1, 'value': 1} + ] + expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows + mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) + with patch('test_pagination_key.KeyTestPager.operation', mockmock): + pager = KeyTestPager(self.client, {'limit': 1}) + with patch( + 'test_pagination_key.KeyTestPager.boundary_func', + lambda p,l: 'test error' if p.id == l.id and p.key == l.key else None): + # Get and assert page + self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) + # Assert has_next True + self.assertTrue(pager.has_next(), 'has_next() should return True.') + with self.assertRaises(Exception): + pager.get_next() + + def test_no_boundary_check_when_no_items_left(self): + mock_rows = [ + {'id': '1', 'key': 1, 'value': 1} + ] + expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows + mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) + with patch('test_pagination_key.KeyTestPager.operation', mockmock): + pager = KeyTestPager(self.client, {'limit': 1}) + with patch( + 'test_pagination_key.KeyTestPager.boundary_func', + Exception('Check boundary should not be called.')): + # Get and assert page if boundary is checked, will raise exception + self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) + # Assert has_next False + self.assertFalse(pager.has_next(), 'has_next() should return True.') From 3d735ad632a14574476e020d66c1e194f4afbd22 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Mar 2025 16:34:23 +0000 Subject: [PATCH 11/27] feat: _BookmarkPager --- ibmcloudant/features/pagination.py | 8 +- .../unit/features/test_pagination_bookmark.py | 151 ++++++++++++++++++ 2 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 test/unit/features/test_pagination_bookmark.py diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 3f0c6419..74bc224e 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -209,13 +209,7 @@ def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse super().__init__(client, operation, ['bookmark'], opts) def _get_next_page_options(self, result: R) -> dict: - pass - - def _get_bookmark(self, result: R): - return result.bookmark - - def _set_bookmark(self, opts:dict, bookmark:str) -> str: - opts['bookmark'] = bookmark + return {'bookmark': result.bookmark} class _AllDocsBasePager(_KeyPager[str]): diff --git a/test/unit/features/test_pagination_bookmark.py b/test/unit/features/test_pagination_bookmark.py new file mode 100644 index 00000000..3b7c9df6 --- /dev/null +++ b/test/unit/features/test_pagination_bookmark.py @@ -0,0 +1,151 @@ +# coding: utf-8 + +# © Copyright IBM Corporation 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable +from itertools import batched +from unittest.mock import Mock, patch +from ibm_cloud_sdk_core import DetailedResponse +from ibmcloudant.cloudant_v1 import SearchResult, SearchResultRow +from ibmcloudant.features.pagination import _BookmarkPager, Pager +from conftest import MockClientBaseCase + +class BookmarkTestPager(_BookmarkPager): + """ + A test subclass of the _BookmarkPager under test. + """ + operation: Callable = None + boundary_func: Callable = lambda p,l: None + + def __init__(self, client, opts): + super().__init__(client, BookmarkTestPager.operation or client.post_view, opts) + + def _result_converter(self) -> Callable[[dict], SearchResult]: + return lambda d: SearchResult.from_dict(d) + + def _items(self, result: SearchResult) -> tuple[SearchResultRow]: + return result.rows + +class MockPageResponses: + """ + Test class for mocking page responses. + """ + def __init__(self, total_items: int, page_size: int): + self.total_items: int = total_items + self.page_size: int = page_size + self.pages = self.generator() + self.expected_pages: list[list[SearchResultRow]] = [] + + def generator(self): + for page in batched(range(0, self.total_items), self.page_size): + rows = [{'id':str(i), 'fields': {'value': i}} for i in page] + yield DetailedResponse(response={'total_rows': self.total_items, 'bookmark': rows[-1]['id'], 'rows': rows}) + yield DetailedResponse(response={'total_rows': self.total_items, 'bookmark': 'last', 'rows': []}) + + def get_next_page(self, **kwargs): + # ignore kwargs + # get next page + page = next(self.pages) + # convert to an expected page + self.expected_pages.append(SearchResult.from_dict(page.get_result()).rows) + return page + + def get_expected_page(self, page: int) -> list[SearchResultRow]: + return self.expected_pages[page - 1] + + def all_expected_items(self) -> list[SearchResultRow]: + all_items: list[SearchResultRow] = [] + for page in self.expected_pages: + all_items.extend(page) + return all_items + +class TestBookmarkPager(MockClientBaseCase): + + # Test page size default + def test_default_page_size(self): + pager: Pager = BookmarkTestPager(self.client, {}) + # Assert the limit default as page size + self.assertEqual(pager._page_size, 200, 'The page size should be one more than the default limit.') + + # Test page size limit + def test_limit_page_size(self): + pager: Pager = BookmarkTestPager(self.client, {'limit': 42}) + # Assert the limit provided as page size + self.assertEqual(pager._page_size, 42, 'The page size should be one more than the default limit.') + + # Test all items on page when no more pages + def test_get_next_page_less_than_limit(self): + page_size = 21 + mock = MockPageResponses(page_size - 1, page_size) + with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): + pager = BookmarkTestPager(self.client, {'limit': page_size}) + # Get and assert first page + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page') + # Assert page size + self.assertEqual(len(actual_page), page_size - 1, 'The actual page size should match the expected page size.') + # Assert has_next False because n+1 limit is 1 more than user page size + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + # Test correct items on page when limit + def test_get_next_page_equal_to_limit(self): + page_size = 14 + mock = MockPageResponses(page_size, page_size) + with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): + pager = BookmarkTestPager(self.client, {'limit': page_size}) + # Get and assert first page + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') + # Assert page size + self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') + # Assert has_next True + self.assertTrue(pager.has_next(), 'has_next() should return True.') + # Assert bookmark + self.assertEqual(pager._next_page_opts['bookmark'], str(page_size - 1), 'The bookmark should be one less than the page size.') + # Get and assert second page + second_page = pager.get_next() + # Note row keys are zero indexed so page size - 1 + self.assertEqual(len(second_page), 0, "The second page should be empty.") + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + # Test correct items on page when n+more + def test_get_next_page_greater_than_limit(self): + page_size = 7 + mock = MockPageResponses(page_size+2, page_size) + with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): + pager = BookmarkTestPager(self.client, {'limit': page_size}) + # Get and assert first page + actual_page = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') + # Assert page size + self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') + # Assert has_next True + self.assertTrue(pager.has_next(), 'has_next() should return True.') + # Get and assert second page + second_page = pager.get_next() + self.assertEqual(len(second_page), 2 , 'The second page should have two items.') + # Note row keys are zero indexed so n+1 element that is first item on second page matches page size + self.assertEqual(second_page[0].id, str(page_size), 'The first item key on the second page should match the page size number.') + self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") + self.assertFalse(pager.has_next(), 'has_next() should return False.') + + # Test getting all items + def test_get_all(self): + page_size = 3 + mock = MockPageResponses(page_size*12, page_size) + with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): + pager = BookmarkTestPager(self.client, {'limit': page_size}) + # Get and assert all items + self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), 'The results should match all the pages.') From 8ccf123a39e2892af84bdbb96c00b7c248ffaab1 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 28 Mar 2025 17:23:23 +0000 Subject: [PATCH 12/27] refactor: pagination entry point factory API --- ibmcloudant/__init__.py | 2 +- ibmcloudant/features/pagination.py | 181 +++++++++++++++++++++++------ 2 files changed, 146 insertions(+), 37 deletions(-) diff --git a/ibmcloudant/__init__.py b/ibmcloudant/__init__.py index 0f2c3f75..1ce2afa9 100644 --- a/ibmcloudant/__init__.py +++ b/ibmcloudant/__init__.py @@ -23,7 +23,7 @@ from .couchdb_session_token_manager import CouchDbSessionTokenManager from .cloudant_v1 import CloudantV1 from .features.changes_follower import ChangesFollower -from .features.pagination import Pager, PagerType +from .features.pagination import Pager, PagerType, Pagination # sdk-core's __construct_authenticator works with a long switch-case so monkey-patching is required get_authenticator.__construct_authenticator = new_construct_authenticator diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 74bc224e..62d51c79 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -15,13 +15,14 @@ """ Feature for paginating requests. - Import :class:`~ibmcloudant.Pager` and :class:`~ibmcloudant.PagerType` + Import :class:`~ibmcloudant.Pagination` and :class:`~ibmcloudant.PagerType` from :mod:`ibmcloudant`. - Use :meth:`Pager.new_pager` to create a :class:`Pager` for different - :class:`PagerType` operations. + Use :meth:`Pagination.new_pagination` to create a :class:`Pagination` + for the specific :class:`PagerType` operation and options. """ + from abc import abstractmethod -from collections.abc import Callable +from collections.abc import Callable, Iterable, Iterator from enum import auto, Enum from functools import partial from types import MappingProxyType @@ -42,6 +43,7 @@ class PagerType(Enum): """ Enumeration of the available Pager types """ + POST_ALL_DOCS = auto() POST_DESIGN_DOCS = auto() POST_FIND = auto() @@ -52,9 +54,10 @@ class PagerType(Enum): POST_SEARCH = auto() POST_VIEW = auto() -class Pager(Protocol[R, I]): +class Pager(Protocol[I]): """ Protocol for pagination of Cloudant operations. + Use Pager.new_pager to create a new pager for one of the operation types in PagerType. """ @@ -64,6 +67,7 @@ def has_next(self) -> bool: """ returns False if there are no more pages """ + raise NotImplementedError() @abstractmethod @@ -71,6 +75,7 @@ def get_next(self) -> tuple[I]: """ returns the next page of results """ + raise NotImplementedError() @abstractmethod @@ -78,19 +83,143 @@ def get_all(self) -> tuple[I]: """ returns all the pages of results in single list """ + raise NotImplementedError() +class Pagination: + """ + Entry point for the pagination features. + + Use :meth:`Pagination.new_pagination` to create a :class:`Pagination` + instance for the specific :class:`PagerType` operation and options. + + Then create a Pager or Iterable using one of the functions: + * :meth:`pager` - for an IBM Cloud SDK style Pager + * :meth:`pages` - for a page Iterable + * :meth:`rows` - for a row Iterable + """ + + def __init__(self, client: CloudantV1, type, opts: dict): + self._client = client + self._operation_type = type + self._initial_opts = dict(opts) + + def pager(self) -> Pager[I]: + """ + Create a new IBM Cloud SDK style Pager. + This type is useful for retrieving one page at a time from a function call. + """ + + return _IteratorPager(self.pages) + + def pages(self) -> Iterable[tuple[I]]: + """ + Create a new Iterable for all the pages. + This type is useful for handling pages in a for loop. + + for page in Pagination.new_pagination(client, **opts).pages(): + ... + """ + + return self._operation_type(self._client, self._initial_opts) + + def rows(self) -> Iterable[I]: + """ + Create a new Iterable for all the rows from all the pages. + This type is useful for handling rows in a for loop. + + for row in Pagination.new_pagination(client, **opts).rows(): + ... + """ + + for page in self.pages(): + yield from page + @classmethod - def new_pager(cls, client:CloudantV1, type: PagerType, **kwargs,): + def new_pagination(cls, client:CloudantV1, type: PagerType, **kwargs): """ - Create a new Pager. + Create a new Pagination. client: CloudantV1 - the Cloudant service client type: PagerType - the operation type to paginate kwargs: dict - the options for the operation """ - raise NotImplementedError() -class _BasePager(Pager): + if type == PagerType.POST_ALL_DOCS: + return Pagination(client, _AllDocsPager, kwargs) + if type == PagerType.POST_DESIGN_DOCS: + return Pagination(client, _DesignDocsPager, kwargs) + if type == PagerType.POST_FIND: + return Pagination(client, _FindPager, kwargs) + if type == PagerType.POST_PARTITION_ALL_DOCS: + return Pagination(client, _AllDocsPartitionPager, kwargs) + if type == PagerType.POST_PARTITION_FIND: + return Pagination(client, _FindPartitionPager, kwargs) + if type == PagerType.POST_PARTITION_SEARCH: + return Pagination(client, _SearchPartitionPager, kwargs) + if type == PagerType.POST_PARTITION_VIEW: + return Pagination(client, _ViewPartitionPager, kwargs) + if type == PagerType.POST_SEARCH: + return Pagination(client, _SearchPager, kwargs) + if type == PagerType.POST_VIEW: + return Pagination(client, _ViewPager, kwargs) + +# TODO state checks +class _IteratorPagerState(Enum): + NEW = auto() + GET_NEXT = auto() + GET_ALL = auto() + CONSUMED = auto() + +class _IteratorPager(Pager[I]): + + _state_mixed_msg = 'This pager has been consumed, use a new Pager.' + _state_consumed_msg = 'Cannot mix get_all() and get_next() use only one method or make a new Pager.' + + def __init__(self, iterable_func: Callable[[], Iterator[tuple[I]]]): + self._iterable_func: Callable[[], Iterator[tuple[I]]] = iterable_func + self._iterator: Iterator[tuple[I]] = iter(self._iterable_func()) + self._state: _IteratorPagerState = _IteratorPagerState.NEW + + def has_next(self) -> bool: + """ + returns False if there are no more pages + """ + + return self._iterator._has_next + + def get_next(self) -> tuple[I]: + """ + returns the next page of results + """ + self._check_state(mode=_IteratorPagerState.GET_NEXT) + page: tuple[I] = next(self._iterator) + if not self._iterator._has_next: + self._state = _IteratorPagerState.CONSUMED + return page + + def get_all(self) -> tuple[I]: + """ + returns all the pages of results in single list + """ + + self._check_state(mode=_IteratorPagerState.GET_ALL) + all_items: list[I] = [] + for page in self._iterable_func(): + all_items.extend(page) + self._state = _IteratorPagerState.CONSUMED + return (*all_items,) + + def _check_state(self, mode: _IteratorPagerState): + if self._state == mode: + return + if self._state == _IteratorPagerState.NEW: + self._state = mode + return + if self._state == _IteratorPagerState.CONSUMED: + raise Exception(_IteratorPager._state_consumed_msg) + raise Exception(_IteratorPager._state_mixed_msg) + +class _BasePager(Iterator[tuple[I]]): def __init__(self, client: CloudantV1, @@ -109,38 +238,18 @@ def __init__(self, for k in page_opts: if v := fixed_opts.pop(k, None): self._next_page_opts[k] = v - self._initial_opts = dict(opts) fixed_opts = MappingProxyType(fixed_opts) # Partial method with the fixed ops self._next_request_function: Callable[..., DetailedResponse] = partial(operation, **fixed_opts) - def _new_copy(self) -> Pager: - # Make and return new instance of the specific sub-class in use - return type(self)( - self._client, - { **self._next_request_function.keywords, **self._initial_opts }) - - - def has_next(self) -> bool: - return self._has_next + def __iter__(self) -> Iterator[tuple[I]]: + return self - def get_next(self) -> tuple[I]: - if self.has_next(): + def __next__(self) -> tuple[I]: + if self._has_next: return (*self._next_request(),) raise StopIteration() - def get_all(self) -> tuple[I]: - all_items = [] - for page in self: - all_items.extend(page) - return (*all_items,) - - def __iter__(self): - return self._new_copy() - - def __next__(self): - return self.get_next() - def _next_request(self) -> list[I]: response: DetailedResponse = self._next_request_function(**self._next_page_opts) result: dict = response.get_result() @@ -177,7 +286,7 @@ def _next_request(self) -> list[I]: if self._boundary_failure is not None: raise Exception(self._boundary_failure) items: list[I] = super()._next_request() - if self.has_next(): + if self._has_next: last_item: I = items.pop() if len(items) > 0: # Get, but don't remove the last item from the list @@ -282,8 +391,8 @@ def _result_converter(self): return ViewResult.from_dict def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: - if penultimate_item.id == (boundary_id:= last_item.id) \ - and penultimate_item.key == (boundary_key:= last_item.key): + if penultimate_item.id == (boundary_id := last_item.id) \ + and penultimate_item.key == (boundary_key := last_item.key): return f'Cannot paginate on a boundary containing identical keys {boundary_key} and document IDs {boundary_id}' return None From 5346aa289a8a384fc8c791b66a2d69e02d63faca Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 1 Apr 2025 09:27:04 +0100 Subject: [PATCH 13/27] refactor: renames --- ibmcloudant/features/pagination.py | 50 +++++++++---------- test/unit/features/test_pagination_base.py | 46 ++++++++--------- .../unit/features/test_pagination_bookmark.py | 20 ++++---- test/unit/features/test_pagination_key.py | 28 +++++------ 4 files changed, 72 insertions(+), 72 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 62d51c79..69806f9a 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -145,23 +145,23 @@ def new_pagination(cls, client:CloudantV1, type: PagerType, **kwargs): """ if type == PagerType.POST_ALL_DOCS: - return Pagination(client, _AllDocsPager, kwargs) + return Pagination(client, _AllDocsPageIterator, kwargs) if type == PagerType.POST_DESIGN_DOCS: - return Pagination(client, _DesignDocsPager, kwargs) + return Pagination(client, _DesignDocsPageIterator, kwargs) if type == PagerType.POST_FIND: - return Pagination(client, _FindPager, kwargs) + return Pagination(client, _FindPageIterator, kwargs) if type == PagerType.POST_PARTITION_ALL_DOCS: - return Pagination(client, _AllDocsPartitionPager, kwargs) + return Pagination(client, _AllDocsPartitionPageIterator, kwargs) if type == PagerType.POST_PARTITION_FIND: - return Pagination(client, _FindPartitionPager, kwargs) + return Pagination(client, _FindPartitionPageIterator, kwargs) if type == PagerType.POST_PARTITION_SEARCH: - return Pagination(client, _SearchPartitionPager, kwargs) + return Pagination(client, _SearchPartitionPageIterator, kwargs) if type == PagerType.POST_PARTITION_VIEW: - return Pagination(client, _ViewPartitionPager, kwargs) + return Pagination(client, _ViewPartitionPageIterator, kwargs) if type == PagerType.POST_SEARCH: - return Pagination(client, _SearchPager, kwargs) + return Pagination(client, _SearchPageIterator, kwargs) if type == PagerType.POST_VIEW: - return Pagination(client, _ViewPager, kwargs) + return Pagination(client, _ViewPageIterator, kwargs) # TODO state checks class _IteratorPagerState(Enum): @@ -219,7 +219,7 @@ def _check_state(self, mode: _IteratorPagerState): raise Exception(_IteratorPager._state_consumed_msg) raise Exception(_IteratorPager._state_mixed_msg) -class _BasePager(Iterator[tuple[I]]): +class _BasePageIterator(Iterator[tuple[I]]): def __init__(self, client: CloudantV1, @@ -276,7 +276,7 @@ def _items(self, result: R) -> list[I]: def _get_next_page_options(self, result: R) -> dict: raise NotImplementedError() -class _KeyPager(_BasePager, Generic[K]): +class _KeyPageIterator(_BasePageIterator, Generic[K]): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): super().__init__(client, operation, ['start_key', 'start_key_doc_id'], opts) @@ -312,7 +312,7 @@ def _page_size_from_opts_limit(self, opts:dict) -> int: def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: raise NotImplementedError() -class _BookmarkPager(_BasePager): +class _BookmarkPageIterator(_BasePageIterator): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): super().__init__(client, operation, ['bookmark'], opts) @@ -320,7 +320,7 @@ def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse def _get_next_page_options(self, result: R) -> dict: return {'bookmark': result.bookmark} -class _AllDocsBasePager(_KeyPager[str]): +class _AllDocsBasePageIterator(_KeyPageIterator[str]): def _result_converter(self) -> Callable[[dict], AllDocsResult]: return AllDocsResult.from_dict @@ -334,22 +334,22 @@ def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: # IDs are always unique in _all_docs pagers so return None return None -class _AllDocsPager(_AllDocsBasePager): +class _AllDocsPageIterator(_AllDocsBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_all_docs, opts) -class _AllDocsPartitionPager(_AllDocsBasePager): +class _AllDocsPartitionPageIterator(_AllDocsBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_partition_all_docs, opts) -class _DesignDocsPager(_AllDocsBasePager): +class _DesignDocsPageIterator(_AllDocsBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_design_docs, opts) -class _FindBasePager(_BookmarkPager): +class _FindBasePageIterator(_BookmarkPageIterator): def _items(self, result: FindResult): return result.docs @@ -357,17 +357,17 @@ def _items(self, result: FindResult): def _result_converter(self): return FindResult.from_dict -class _FindPager(_FindBasePager): +class _FindPageIterator(_FindBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_find, opts) -class _FindPartitionPager(_FindBasePager): +class _FindPartitionPageIterator(_FindBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_partition_find, opts) -class _SearchBasePager(_BookmarkPager): +class _SearchBasePageIterator(_BookmarkPageIterator): def _items(self, result: SearchResult): return result.rows @@ -375,17 +375,17 @@ def _items(self, result: SearchResult): def _result_converter(self): return SearchResult.from_dict -class _SearchPager(_SearchBasePager): +class _SearchPageIterator(_SearchBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_search, opts) -class _SearchPartitionPager(_SearchBasePager): +class _SearchPartitionPageIterator(_SearchBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_partition_search, opts) -class _ViewBasePager(_KeyPager[any]): +class _ViewBasePageIterator(_KeyPageIterator[any]): def _result_converter(self): return ViewResult.from_dict @@ -396,12 +396,12 @@ def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: return f'Cannot paginate on a boundary containing identical keys {boundary_key} and document IDs {boundary_id}' return None -class _ViewPager(_ViewBasePager): +class _ViewPageIterator(_ViewBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_view, opts) -class _ViewPartitionPager(_ViewBasePager): +class _ViewPartitionPageIterator(_ViewBasePageIterator): def __init__(self, client: CloudantV1, opts: dict): super().__init__(client, client.post_partition_view, opts) diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py index 467571e5..8f525246 100644 --- a/test/unit/features/test_pagination_base.py +++ b/test/unit/features/test_pagination_base.py @@ -19,10 +19,10 @@ from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow -from ibmcloudant.features.pagination import _BasePager, Pager +from ibmcloudant.features.pagination import _BasePageIterator, Pager from conftest import MockClientBaseCase -class TestPager(_BasePager): +class TestPageIterator(_BasePageIterator): """ A test subclass of the _BasePager under test. """ @@ -30,7 +30,7 @@ class TestPager(_BasePager): page_keys: list[str] = [] def __init__(self, client, opts): - super().__init__(client, TestPager.operation or client.post_view, TestPager.page_keys, opts) + super().__init__(client, TestPageIterator.operation or client.post_view, TestPageIterator.page_keys, opts) def _result_converter(self) -> Callable[[dict], ViewResult]: return lambda d: ViewResult.from_dict(d) @@ -77,11 +77,11 @@ def all_expected_items(self) -> list[ViewResultRow]: all_items.extend(page) return all_items -class TestBasePager(MockClientBaseCase): +class TestBasePageIterator(MockClientBaseCase): def test_init(self): operation = self.client.post_view opts = {'db': 'test', 'limit': 20} - pager: Pager = TestPager(self.client, opts) + pager: Pager = TestPageIterator(self.client, opts) # Assert client is set self.assertEqual(pager._client, self.client, 'The supplied client should be set.') # Assert operation is set @@ -96,7 +96,7 @@ def test_partial_options(self): opts = {**static_opts, **page_opts} # Use page_opts.keys() to pass the list of names for page options with patch('test_pagination_base.TestPager.page_keys', page_opts.keys()): - pager: Pager = TestPager(self.client, opts) + pager: Pager = TestPageIterator(self.client, opts) # Assert partial function has only static opts self.assertEqual(pager._next_request_function.keywords, static_opts, 'The partial function kwargs should be only the static options.') # Assert next page options @@ -104,7 +104,7 @@ def test_partial_options(self): def test_default_page_size(self): opts = {'db': 'test'} - pager: Pager = TestPager(self.client, opts) + pager: Pager = TestPageIterator(self.client, opts) # Assert the default page size expected_page_size = 200 self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') @@ -112,7 +112,7 @@ def test_default_page_size(self): def test_limit_page_size(self): opts = {'db': 'test', 'limit': 42} - pager: Pager = TestPager(self.client, opts) + pager: Pager = TestPageIterator(self.client, opts) # Assert the provided page size expected_page_size = 42 self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') @@ -120,7 +120,7 @@ def test_limit_page_size(self): def test_has_next_initially_true(self): opts = {'limit': 1} - pager: Pager = TestPager(self.client, opts) + pager: Pager = TestPageIterator(self.client, opts) # Assert has_next() self.assertTrue(pager.has_next(), 'has_next() should initially return True.') @@ -128,7 +128,7 @@ def test_has_next_true_for_result_equal_to_limit(self): page_size = 1 # Init with mock that returns only a single row with patch('test_pagination_base.TestPager.operation', MockPageResponses(1, page_size).get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get first page with 1 result @@ -140,7 +140,7 @@ def test_has_next_false_for_result_less_than_limit(self): page_size = 1 # Init with mock that returns zero rows with patch('test_pagination_base.TestPager.operation', MockPageResponses(0, page_size).get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get first page with 0 result @@ -153,7 +153,7 @@ def test_get_next_first_page(self): # Mock that returns one page of 25 items mock = MockPageResponses(page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get first page @@ -166,7 +166,7 @@ def test_get_next_two_pages(self): # Mock that returns two pages of 3 items mock = MockPageResponses(2*page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get first page @@ -187,7 +187,7 @@ def test_get_next_until_empty(self): # Mock that returns 3 pages of 3 items mock = MockPageResponses(3*page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) page_count = 0 @@ -206,7 +206,7 @@ def test_get_next_until_smaller(self): # Mock that returns 3 pages of 3 items, then 1 more page with 1 item mock = MockPageResponses(3*page_size + 1, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) page_count = 0 @@ -225,7 +225,7 @@ def test_get_next_exception(self): # Mock that returns one page of one item mock = MockPageResponses(page_size - 1, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get first and only page @@ -243,7 +243,7 @@ def test_get_all(self): # Mock that returns 6 pages of 11 items, then 1 more page with 5 items mock = MockPageResponses(71, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) actual_items = pager.get_all() @@ -254,7 +254,7 @@ def test_iter_next_first_page(self): # Mock that returns two pages of 7 items mock = MockPageResponses(2*page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get first page @@ -265,7 +265,7 @@ def test_iter(self): page_size = 23 mock = MockPageResponses(3*page_size-1, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Check pager is an iterator @@ -280,7 +280,7 @@ def test_pages_immutable(self): page_size = 1 mock = MockPageResponses(page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) # Get page @@ -292,7 +292,7 @@ def test_set_next_page_options(self): page_size = 1 mock = MockPageResponses(5*page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") @@ -310,7 +310,7 @@ def test_get_next_resumes_after_error(self): page_size = 1 mock = MockPageResponses(3*page_size, page_size) with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") @@ -339,7 +339,7 @@ def test_get_all_restarts_after_error(self): mock.get_next_page() ]) with patch('test_pagination_base.TestPager.operation', mockmock): - pager: Pager = TestPager( + pager: Pager = TestPageIterator( self.client, {'limit': page_size}) with self.assertRaises(Exception): diff --git a/test/unit/features/test_pagination_bookmark.py b/test/unit/features/test_pagination_bookmark.py index 3b7c9df6..47a0ef85 100644 --- a/test/unit/features/test_pagination_bookmark.py +++ b/test/unit/features/test_pagination_bookmark.py @@ -19,10 +19,10 @@ from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import SearchResult, SearchResultRow -from ibmcloudant.features.pagination import _BookmarkPager, Pager +from ibmcloudant.features.pagination import _BookmarkPageIterator, Pager from conftest import MockClientBaseCase -class BookmarkTestPager(_BookmarkPager): +class BookmarkTestPageIterator(_BookmarkPageIterator): """ A test subclass of the _BookmarkPager under test. """ @@ -30,7 +30,7 @@ class BookmarkTestPager(_BookmarkPager): boundary_func: Callable = lambda p,l: None def __init__(self, client, opts): - super().__init__(client, BookmarkTestPager.operation or client.post_view, opts) + super().__init__(client, BookmarkTestPageIterator.operation or client.post_view, opts) def _result_converter(self) -> Callable[[dict], SearchResult]: return lambda d: SearchResult.from_dict(d) @@ -71,17 +71,17 @@ def all_expected_items(self) -> list[SearchResultRow]: all_items.extend(page) return all_items -class TestBookmarkPager(MockClientBaseCase): +class TestBookmarkPageIterator(MockClientBaseCase): # Test page size default def test_default_page_size(self): - pager: Pager = BookmarkTestPager(self.client, {}) + pager: Pager = BookmarkTestPageIterator(self.client, {}) # Assert the limit default as page size self.assertEqual(pager._page_size, 200, 'The page size should be one more than the default limit.') # Test page size limit def test_limit_page_size(self): - pager: Pager = BookmarkTestPager(self.client, {'limit': 42}) + pager: Pager = BookmarkTestPageIterator(self.client, {'limit': 42}) # Assert the limit provided as page size self.assertEqual(pager._page_size, 42, 'The page size should be one more than the default limit.') @@ -90,7 +90,7 @@ def test_get_next_page_less_than_limit(self): page_size = 21 mock = MockPageResponses(page_size - 1, page_size) with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPager(self.client, {'limit': page_size}) + pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page actual_page = pager.get_next() self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page') @@ -104,7 +104,7 @@ def test_get_next_page_equal_to_limit(self): page_size = 14 mock = MockPageResponses(page_size, page_size) with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPager(self.client, {'limit': page_size}) + pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page actual_page = pager.get_next() self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') @@ -125,7 +125,7 @@ def test_get_next_page_greater_than_limit(self): page_size = 7 mock = MockPageResponses(page_size+2, page_size) with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPager(self.client, {'limit': page_size}) + pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page actual_page = pager.get_next() self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') @@ -146,6 +146,6 @@ def test_get_all(self): page_size = 3 mock = MockPageResponses(page_size*12, page_size) with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPager(self.client, {'limit': page_size}) + pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert all items self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), 'The results should match all the pages.') diff --git a/test/unit/features/test_pagination_key.py b/test/unit/features/test_pagination_key.py index e081877b..b843a9ce 100644 --- a/test/unit/features/test_pagination_key.py +++ b/test/unit/features/test_pagination_key.py @@ -19,10 +19,10 @@ from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow -from ibmcloudant.features.pagination import _KeyPager, Pager +from ibmcloudant.features.pagination import _KeyPageIterator, Pager from conftest import MockClientBaseCase -class KeyTestPager(_KeyPager): +class KeyTestPageIterator(_KeyPageIterator): """ A test subclass of the _KeyPager under test. """ @@ -30,7 +30,7 @@ class KeyTestPager(_KeyPager): boundary_func: Callable = lambda p,l: None def __init__(self, client, opts): - super().__init__(client, KeyTestPager.operation or client.post_view, opts) + super().__init__(client, KeyTestPageIterator.operation or client.post_view, opts) def _result_converter(self) -> Callable[[dict], ViewResult]: return lambda d: ViewResult.from_dict(d) @@ -45,7 +45,7 @@ def _get_next_page_options(self, result: ViewResult) -> dict: return {'start_key': result.rows[-1].key} def check_boundary(self, penultimate_item, last_item): - return KeyTestPager.boundary_func(penultimate_item, last_item) + return KeyTestPageIterator.boundary_func(penultimate_item, last_item) class MockPageResponses: """ @@ -87,17 +87,17 @@ def all_expected_items(self) -> list[ViewResultRow]: all_items.extend(page) return all_items -class TestKeyPager(MockClientBaseCase): +class TestKeyPageIterator(MockClientBaseCase): # Test page size default (+1) def test_default_page_size(self): - pager: Pager = KeyTestPager(self.client, {}) + pager: Pager = KeyTestPageIterator(self.client, {}) # Assert the limit default as page size self.assertEqual(pager._page_size, 201, 'The page size should be one more than the default limit.') # Test page size limit (+1) def test_limit_page_size(self): - pager: Pager = KeyTestPager(self.client, {'limit': 42}) + pager: Pager = KeyTestPageIterator(self.client, {'limit': 42}) # Assert the limit provided as page size self.assertEqual(pager._page_size, 43, 'The page size should be one more than the default limit.') @@ -106,7 +106,7 @@ def test_get_next_page_less_than_limit(self): page_size = 21 mock = MockPageResponses(page_size, page_size) with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPager(self.client, {'limit': page_size}) + pager = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page actual_page = pager.get_next() self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page') @@ -120,7 +120,7 @@ def test_get_next_page_equal_to_limit(self): page_size = 14 mock = MockPageResponses(page_size+1, page_size) with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPager(self.client, {'limit': page_size}) + pager = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page actual_page = pager.get_next() self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') @@ -141,7 +141,7 @@ def test_get_next_page_greater_than_limit(self): page_size = 7 mock = MockPageResponses(page_size+2, page_size) with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPager(self.client, {'limit': page_size}) + pager = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page actual_page = pager.get_next() self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') @@ -162,7 +162,7 @@ def test_get_all(self): page_size = 3 mock = MockPageResponses(page_size*12, page_size) with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPager(self.client, {'limit': page_size}) + pager = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert all items self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), 'The results should match all the pages.') @@ -174,7 +174,7 @@ def test_no_boundary_check_by_default(self): expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) with patch('test_pagination_key.KeyTestPager.operation', mockmock): - pager = KeyTestPager(self.client, {'limit': 1}) + pager = KeyTestPageIterator(self.client, {'limit': 1}) # Get and assert page self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) @@ -186,7 +186,7 @@ def test_boundary_failure_throws_on_get_next(self): expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) with patch('test_pagination_key.KeyTestPager.operation', mockmock): - pager = KeyTestPager(self.client, {'limit': 1}) + pager = KeyTestPageIterator(self.client, {'limit': 1}) with patch( 'test_pagination_key.KeyTestPager.boundary_func', lambda p,l: 'test error' if p.id == l.id and p.key == l.key else None): @@ -204,7 +204,7 @@ def test_no_boundary_check_when_no_items_left(self): expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) with patch('test_pagination_key.KeyTestPager.operation', mockmock): - pager = KeyTestPager(self.client, {'limit': 1}) + pager = KeyTestPageIterator(self.client, {'limit': 1}) with patch( 'test_pagination_key.KeyTestPager.boundary_func', Exception('Check boundary should not be called.')): From 5f46857fd3276c3bade3a444942b018148d99e73 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 1 Apr 2025 11:22:48 +0100 Subject: [PATCH 14/27] test: updates for new entry point and renames --- test/unit/features/test_pagination_base.py | 333 +++++++++++------- .../unit/features/test_pagination_bookmark.py | 51 +-- test/unit/features/test_pagination_key.py | 77 ++-- 3 files changed, 272 insertions(+), 189 deletions(-) diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py index 8f525246..2071fd76 100644 --- a/test/unit/features/test_pagination_base.py +++ b/test/unit/features/test_pagination_base.py @@ -16,10 +16,11 @@ from collections.abc import Callable from itertools import batched +from typing import Iterable from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow -from ibmcloudant.features.pagination import _BasePageIterator, Pager +from ibmcloudant.features.pagination import _BasePageIterator, _IteratorPager, Pager, Pagination from conftest import MockClientBaseCase class TestPageIterator(_BasePageIterator): @@ -70,7 +71,7 @@ def get_next_page(self, **kwargs): def get_expected_page(self, page: int) -> list[ViewResultRow]: return self.expected_pages[page - 1] - + def all_expected_items(self) -> list[ViewResultRow]: all_items: list[ViewResultRow] = [] for page in self.expected_pages: @@ -81,252 +82,263 @@ class TestBasePageIterator(MockClientBaseCase): def test_init(self): operation = self.client.post_view opts = {'db': 'test', 'limit': 20} - pager: Pager = TestPageIterator(self.client, opts) + page_iterator: Iterable[ViewResultRow] = TestPageIterator(self.client, opts) # Assert client is set - self.assertEqual(pager._client, self.client, 'The supplied client should be set.') + self.assertEqual(page_iterator._client, self.client, 'The supplied client should be set.') # Assert operation is set - self.assertIsNotNone(pager._next_request_function, 'The operation function should be set.') + self.assertIsNotNone(page_iterator._next_request_function, 'The operation function should be set.') # Assert partial function parts are as expected - self.assertEqual(pager._next_request_function.func, operation, 'The partial function should be the operation.') - self.assertEqual(pager._next_request_function.keywords, opts, 'The partial function kwargs should be the options.') + self.assertEqual(page_iterator._next_request_function.func, operation, 'The partial function should be the operation.') + self.assertEqual(page_iterator._next_request_function.keywords, opts, 'The partial function kwargs should be the options.') def test_partial_options(self): static_opts = {'db': 'test', 'limit': 20, 'baz': 'faz'} page_opts = {'foo': 'boo', 'bar': 'far'} opts = {**static_opts, **page_opts} # Use page_opts.keys() to pass the list of names for page options - with patch('test_pagination_base.TestPager.page_keys', page_opts.keys()): - pager: Pager = TestPageIterator(self.client, opts) + with patch('test_pagination_base.TestPageIterator.page_keys', page_opts.keys()): + page_iterator: Iterable[ViewResultRow] = TestPageIterator(self.client, opts) # Assert partial function has only static opts - self.assertEqual(pager._next_request_function.keywords, static_opts, 'The partial function kwargs should be only the static options.') + self.assertEqual(page_iterator._next_request_function.keywords, static_opts, 'The partial function kwargs should be only the static options.') # Assert next page options - self.assertEqual(pager._next_page_opts, page_opts, 'The next page options should match the expected.') + self.assertEqual(page_iterator._next_page_opts, page_opts, 'The next page options should match the expected.') def test_default_page_size(self): opts = {'db': 'test'} - pager: Pager = TestPageIterator(self.client, opts) + page_iterator: Iterable[ViewResultRow] = TestPageIterator(self.client, opts) # Assert the default page size expected_page_size = 200 - self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') - self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') + self.assertEqual(page_iterator._page_size, expected_page_size, 'The default page size should be set.') + self.assertEqual(page_iterator._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') def test_limit_page_size(self): opts = {'db': 'test', 'limit': 42} - pager: Pager = TestPageIterator(self.client, opts) + page_iterator: Iterable[ViewResultRow] = TestPageIterator(self.client, opts) # Assert the provided page size expected_page_size = 42 - self.assertEqual(pager._page_size, expected_page_size, 'The default page size should be set.') - self.assertEqual(pager._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') + self.assertEqual(page_iterator._page_size, expected_page_size, 'The default page size should be set.') + self.assertEqual(page_iterator._next_request_function.keywords, opts | {'limit': expected_page_size}, 'The default page size should be present in the options.') def test_has_next_initially_true(self): opts = {'limit': 1} - pager: Pager = TestPageIterator(self.client, opts) - # Assert has_next() - self.assertTrue(pager.has_next(), 'has_next() should initially return True.') + page_iterator: Iterable[ViewResultRow] = TestPageIterator(self.client, opts) + # Assert _has_next + self.assertTrue(page_iterator._has_next, '_has_next should initially return True.') def test_has_next_true_for_result_equal_to_limit(self): page_size = 1 # Init with mock that returns only a single row - with patch('test_pagination_base.TestPager.operation', MockPageResponses(1, page_size).get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', MockPageResponses(1, page_size).get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) # Get first page with 1 result - pager.get_next() - # Assert has_next() - self.assertTrue(pager.has_next(), 'has_next() should return True.') + next(page_iterator) + # Assert _has_next + self.assertTrue(page_iterator._has_next, '_has_next should return True.') def test_has_next_false_for_result_less_than_limit(self): page_size = 1 # Init with mock that returns zero rows - with patch('test_pagination_base.TestPager.operation', MockPageResponses(0, page_size).get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', MockPageResponses(0, page_size).get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) # Get first page with 0 result - pager.get_next() - # Assert has_next() - self.assertFalse(pager.has_next(), 'has_next() should return False.') + next(page_iterator) + # Assert _has_next + self.assertFalse(page_iterator._has_next, '_has_next should return False.') - def test_get_next_first_page(self): + def test_next_first_page(self): page_size = 25 # Mock that returns one page of 25 items mock = MockPageResponses(page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) # Get first page - actual_page: list[ViewResultRow] = pager.get_next() + actual_page: list[ViewResultRow] = next(page_iterator) # Assert first page self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") - def test_get_next_two_pages(self): + def test_next_two_pages(self): page_size = 3 # Mock that returns two pages of 3 items mock = MockPageResponses(2*page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) # Get first page - actual_page_1: list[ViewResultRow] = pager.get_next() + actual_page_1: list[ViewResultRow] = next(page_iterator) # Assert first page self.assertSequenceEqual(actual_page_1, mock.get_expected_page(1), "The actual page should match the expected page") # Assert has_next - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') # Get second page - actual_page_2: list[ViewResultRow] = pager.get_next() + actual_page_2: list[ViewResultRow] = next(page_iterator) # Assert first page self.assertSequenceEqual(actual_page_2, mock.get_expected_page(2), "The actual page should match the expected page") # Assert has_next, True since page is not smaller than limit - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') - def test_get_next_until_empty(self): + def test_next_until_empty(self): page_size = 3 # Mock that returns 3 pages of 3 items mock = MockPageResponses(3*page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) page_count = 0 actual_items = [] - while pager.has_next(): + while page_iterator._has_next: page_count += 1 - page = pager.get_next() + page = next(page_iterator) # Assert each page is the same or smaller than the limit to confirm all results not in one page self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") actual_items.extend(page) self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") - def test_get_next_until_smaller(self): + def test_next_until_smaller(self): page_size = 3 # Mock that returns 3 pages of 3 items, then 1 more page with 1 item mock = MockPageResponses(3*page_size + 1, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) page_count = 0 actual_items = [] - while pager.has_next(): + while page_iterator._has_next: page_count += 1 - page = pager.get_next() + page = next(page_iterator) # Assert each page is the same or smaller than the limit to confirm all results not in one page self.assertTrue(len(page) <= page_size, "The actual page size should be the expected page size.") actual_items.extend(page) self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") self.assertEqual(page_count, len(mock.expected_pages), "There should be the correct number of pages.") - def test_get_next_exception(self): + def test_next_exception(self): page_size = 2 # Mock that returns one page of one item mock = MockPageResponses(page_size - 1, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) # Get first and only page - actual_page: list[ViewResultRow] = pager.get_next() + actual_page: list[ViewResultRow] = next(page_iterator) # Assert page self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") - # Assert has_next() now False - self.assertFalse(pager.has_next(), 'has_next() should return False.') + # Assert _has_next now False + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Assert StopIteraton on get_next() with self.assertRaises(StopIteration): - pager.get_next() - - def test_get_all(self): - page_size = 11 - # Mock that returns 6 pages of 11 items, then 1 more page with 5 items - mock = MockPageResponses(71, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( - self.client, - {'limit': page_size}) - actual_items = pager.get_all() - self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") - - def test_iter_next_first_page(self): - page_size = 7 - # Mock that returns two pages of 7 items - mock = MockPageResponses(2*page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( - self.client, - {'limit': page_size}) - # Get first page - actual_page: list[ViewResultRow] = next(pager) - self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") - - def test_iter(self): - page_size = 23 - mock = MockPageResponses(3*page_size-1, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( - self.client, - {'limit': page_size}) - # Check pager is an iterator - page_number = 0 - for page in pager: - page_number += 1 - self.assertSequenceEqual(page, mock.get_expected_page(page_number), "The actual page should match the expected page") - # Asser the correct number of pages - self.assertEqual(page_number, 3, 'There should have been 3 pages.') + next(page_iterator) def test_pages_immutable(self): page_size = 1 mock = MockPageResponses(page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) # Get page - actual_page: list[ViewResultRow] = pager.get_next() + actual_page: list[ViewResultRow] = next(page_iterator) # Assert immutable tuple type self.assertIsInstance(actual_page, tuple) def test_set_next_page_options(self): page_size = 1 mock = MockPageResponses(5*page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) - self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") + self.assertIsNone(page_iterator._next_page_opts.get('start_key'), "The start key should intially be None.") # Since we use a page size of 1, each next page options key, is the same as the element from the page and the page count page_count = 0 - while pager.has_next(): - page = pager.get_next() - if pager.has_next(): - self.assertEqual(page_count, pager._next_page_opts.get('start_key'), "The key should increment per page.") + while page_iterator._has_next: + page = next(page_iterator) + if page_iterator._has_next: + self.assertEqual(page_count, page_iterator._next_page_opts.get('start_key'), "The key should increment per page.") else: - self.assertEqual(page_count - 1, pager._next_page_opts.get('start_key'), "The options should not be set for the final page.") + self.assertEqual(page_count - 1, page_iterator._next_page_opts.get('start_key'), "The options should not be set for the final page.") page_count += 1 - def test_get_next_resumes_after_error(self): + def test_next_resumes_after_error(self): page_size = 1 mock = MockPageResponses(3*page_size, page_size) - with patch('test_pagination_base.TestPager.operation', mock.get_next_page): - pager: Pager = TestPageIterator( + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) - self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should intially be None.") - actual_page = pager.get_next() + self.assertIsNone(page_iterator._next_page_opts.get('start_key'), "The start key should intially be None.") + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") - self.assertEqual(0, pager._next_page_opts.get('start_key'), "The start_key should be 0 for the second page.") - with patch('ibmcloudant.features.pagination._BasePager._next_request', Exception('test exception')): + self.assertEqual(0, page_iterator._next_page_opts.get('start_key'), "The start_key should be 0 for the second page.") + with patch('ibmcloudant.features.pagination._BasePageIterator._next_request', Exception('test exception')): with self.assertRaises(Exception): - pager.get_next() - self.assertTrue(pager.has_next(), 'has_next() should return True.') - self.assertEqual(0, pager._next_page_opts.get('start_key'), "The start_key should still be 0 for the second page.") - second_page = pager.get_next() + next(page_iterator) + self.assertTrue(page_iterator._has_next, '_has_next should return True.') + self.assertEqual(0, page_iterator._next_page_opts.get('start_key'), "The start_key should still be 0 for the second page.") + second_page = next(page_iterator) self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") - self.assertTrue(pager.has_next(), 'has_next() should return False.') + self.assertTrue(page_iterator._has_next, '_has_next should return False.') + + + def test_pages_iterable(self): + page_size = 23 + mock = MockPageResponses(3*page_size-1, page_size) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + # Check pages are iterable + page_number = 0 + for page in pagination.pages(): + page_number += 1 + self.assertSequenceEqual(page, mock.get_expected_page(page_number), "The actual page should match the expected page") + # Asser the correct number of pages + self.assertEqual(page_number, 3, 'There should have been 3 pages.') + + def test_rows_iterable(self): + page_size = 23 + mock = MockPageResponses(3*page_size-1, page_size) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + actual_items = [] + # Check rows are iterable + for row in pagination.rows(): + actual_items.append(row) + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The actual rows should match the expected rows.") + + def test_as_pager_get_next_first_page(self): + page_size = 7 + # Mock that returns two pages of 7 items + mock = MockPageResponses(2*page_size, page_size) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + pager = pagination.pager() + # Get first page + actual_page: list[ViewResultRow] = pager.get_next() + self.assertSequenceEqual(actual_page, mock.get_expected_page(1), "The actual page should match the expected page") - def test_get_all_restarts_after_error(self): + def test_as_pager_get_all(self): + page_size = 11 + # Mock that returns 6 pages of 11 items, then 1 more page with 5 items + mock = MockPageResponses(71, page_size) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + pager: Pager[ViewResultRow] = pagination.pager() + actual_items = pager.get_all() + self.assertSequenceEqual(actual_items, mock.all_expected_items(), "The results should match all the pages.") + # Assert consumed state prevents calling again + with self.assertRaises(Exception, msg=_IteratorPager._state_consumed_msg): + pager.get_all() + + def test_as_pager_get_all_restarts_after_error(self): page_size = 1 mock = MockPageResponses(2*page_size, page_size) first_page = mock.get_next_page() @@ -338,12 +350,81 @@ def test_get_all_restarts_after_error(self): first_page, mock.get_next_page() ]) - with patch('test_pagination_base.TestPager.operation', mockmock): - pager: Pager = TestPageIterator( - self.client, - {'limit': page_size}) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mockmock): + pager = pagination.pager() with self.assertRaises(Exception): pager.get_all() - # After the error we should not have a start_key set - self.assertIsNone(pager._next_page_opts.get('start_key'), "The start key should be None.") self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), "The results should match all the pages.") + + def test_as_pager_get_next_get_all_throws(self): + page_size = 11 + # Mock that returns 6 pages of 11 items, then 1 more page with 5 items + mock = MockPageResponses(71, page_size) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + pager: Pager[ViewResultRow] = pagination.pager() + first_page = pager.get_next() + self.assertSequenceEqual(first_page, mock.get_expected_page(1), "The actual page should match the expected page") + # Assert throws + with self.assertRaises(Exception, msg=_IteratorPager._state_mixed_msg): + pager.get_all() + # Assert second page ok + self.assertSequenceEqual(pager.get_next(), mock.get_expected_page(2), "The actual page should match the expected page") + + def test_as_pager_get_all_get_next_throws(self): + page_size = 1 + mock = MockPageResponses(2*page_size, page_size) + first_page = mock.get_next_page() + # mock response order + # first page, error, first page replay, second page + mockmock = Mock(side_effect=[ + first_page, + Exception('test exception') + ]) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mockmock): + pager = pagination.pager() + # Stop get all part way through so it isn't consumed when we call get Next + with self.assertRaises(Exception): + pager.get_all() + # Assert calling get_next() throws + with self.assertRaises(Exception, msg=_IteratorPager._state_mixed_msg): + pager.get_next() + + def test_as_pager_get_next_resumes_after_error(self): + page_size = 1 + mock = MockPageResponses(2*page_size, page_size) + # mock response order + # first page, error, second page + mockmock = Mock(side_effect=[ + mock.get_next_page(), + Exception('test exception'), + mock.get_next_page() + ]) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mockmock): + pager = pagination.pager() + # Assert first page + self.assertSequenceEqual(pager.get_next(), mock.get_expected_page(1), "The actual page should match the expected page") + with self.assertRaises(Exception): + pager.get_next() + # Assert second page after error + self.assertSequenceEqual(pager.get_next(), mock.get_expected_page(2), "The actual page should match the expected page") + + def test_as_pager_get_next_until_consumed(self): + page_size = 7 + # Mock that returns two pages of 7 items + mock = MockPageResponses(2*page_size, page_size) + pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) + with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): + pager = pagination.pager() + page_count = 0 + while pager.has_next(): + page_count += 1 + self.assertSequenceEqual(pager.get_next(), mock.get_expected_page(page_count), "The actual page should match the expected page") + # Note 3 because third page is empty + self.assertEqual(page_count, 3, 'There should be the expected number of pages.') + # Assert consumed state prevents calling again + with self.assertRaises(Exception, msg=_IteratorPager._state_consumed_msg): + pager.get_next() diff --git a/test/unit/features/test_pagination_bookmark.py b/test/unit/features/test_pagination_bookmark.py index 47a0ef85..a28de666 100644 --- a/test/unit/features/test_pagination_bookmark.py +++ b/test/unit/features/test_pagination_bookmark.py @@ -14,12 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections.abc import Callable +from collections.abc import Callable, Iterator from itertools import batched from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import SearchResult, SearchResultRow -from ibmcloudant.features.pagination import _BookmarkPageIterator, Pager +from ibmcloudant.features.pagination import _BookmarkPageIterator, Pager, Pagination from conftest import MockClientBaseCase class BookmarkTestPageIterator(_BookmarkPageIterator): @@ -75,77 +75,78 @@ class TestBookmarkPageIterator(MockClientBaseCase): # Test page size default def test_default_page_size(self): - pager: Pager = BookmarkTestPageIterator(self.client, {}) + page_iterator: Iterator[tuple[SearchResultRow]] = BookmarkTestPageIterator(self.client, {}) # Assert the limit default as page size - self.assertEqual(pager._page_size, 200, 'The page size should be one more than the default limit.') + self.assertEqual(page_iterator._page_size, 200, 'The page size should be one more than the default limit.') # Test page size limit def test_limit_page_size(self): - pager: Pager = BookmarkTestPageIterator(self.client, {'limit': 42}) + page_iterator: Iterator[tuple[SearchResultRow]] = BookmarkTestPageIterator(self.client, {'limit': 42}) # Assert the limit provided as page size - self.assertEqual(pager._page_size, 42, 'The page size should be one more than the default limit.') + self.assertEqual(page_iterator._page_size, 42, 'The page size should be one more than the default limit.') # Test all items on page when no more pages def test_get_next_page_less_than_limit(self): page_size = 21 mock = MockPageResponses(page_size - 1, page_size) - with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) + with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): + page_iterator = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page - actual_page = pager.get_next() + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page') # Assert page size self.assertEqual(len(actual_page), page_size - 1, 'The actual page size should match the expected page size.') # Assert has_next False because n+1 limit is 1 more than user page size - self.assertFalse(pager.has_next(), 'has_next() should return False.') + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test correct items on page when limit def test_get_next_page_equal_to_limit(self): page_size = 14 mock = MockPageResponses(page_size, page_size) - with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) + with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): + page_iterator = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page - actual_page = pager.get_next() + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') # Assert page size self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') # Assert has_next True - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') # Assert bookmark - self.assertEqual(pager._next_page_opts['bookmark'], str(page_size - 1), 'The bookmark should be one less than the page size.') + self.assertEqual(page_iterator._next_page_opts['bookmark'], str(page_size - 1), 'The bookmark should be one less than the page size.') # Get and assert second page - second_page = pager.get_next() + second_page = next(page_iterator) # Note row keys are zero indexed so page size - 1 self.assertEqual(len(second_page), 0, "The second page should be empty.") - self.assertFalse(pager.has_next(), 'has_next() should return False.') + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test correct items on page when n+more def test_get_next_page_greater_than_limit(self): page_size = 7 mock = MockPageResponses(page_size+2, page_size) - with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) + with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): + page_iterator = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page - actual_page = pager.get_next() + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') # Assert page size self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') # Assert has_next True - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') # Get and assert second page - second_page = pager.get_next() + second_page = next(page_iterator) self.assertEqual(len(second_page), 2 , 'The second page should have two items.') # Note row keys are zero indexed so n+1 element that is first item on second page matches page size self.assertEqual(second_page[0].id, str(page_size), 'The first item key on the second page should match the page size number.') self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") - self.assertFalse(pager.has_next(), 'has_next() should return False.') + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test getting all items def test_get_all(self): page_size = 3 mock = MockPageResponses(page_size*12, page_size) - with patch('test_pagination_bookmark.BookmarkTestPager.operation', mock.get_next_page): - pager = BookmarkTestPageIterator(self.client, {'limit': page_size}) + pagination = Pagination(self.client, BookmarkTestPageIterator, {'limit': page_size}) + with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): + pager = pagination.pager() # Get and assert all items self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), 'The results should match all the pages.') diff --git a/test/unit/features/test_pagination_key.py b/test/unit/features/test_pagination_key.py index b843a9ce..c9a16215 100644 --- a/test/unit/features/test_pagination_key.py +++ b/test/unit/features/test_pagination_key.py @@ -14,12 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections.abc import Callable +from collections.abc import Callable, Iterator from itertools import batched from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow -from ibmcloudant.features.pagination import _KeyPageIterator, Pager +from ibmcloudant.features.pagination import _KeyPageIterator, Pager, Pagination from conftest import MockClientBaseCase class KeyTestPageIterator(_KeyPageIterator): @@ -91,78 +91,79 @@ class TestKeyPageIterator(MockClientBaseCase): # Test page size default (+1) def test_default_page_size(self): - pager: Pager = KeyTestPageIterator(self.client, {}) + page_iterator: Iterator[tuple[ViewResultRow]] = KeyTestPageIterator(self.client, {}) # Assert the limit default as page size - self.assertEqual(pager._page_size, 201, 'The page size should be one more than the default limit.') + self.assertEqual(page_iterator._page_size, 201, 'The page size should be one more than the default limit.') # Test page size limit (+1) def test_limit_page_size(self): - pager: Pager = KeyTestPageIterator(self.client, {'limit': 42}) + page_iterator: Iterator[tuple[ViewResultRow]] = KeyTestPageIterator(self.client, {'limit': 42}) # Assert the limit provided as page size - self.assertEqual(pager._page_size, 43, 'The page size should be one more than the default limit.') + self.assertEqual(page_iterator._page_size, 43, 'The page size should be one more than the default limit.') # Test all items on page when no more pages def test_get_next_page_less_than_limit(self): page_size = 21 mock = MockPageResponses(page_size, page_size) - with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPageIterator(self.client, {'limit': page_size}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): + page_iterator = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page - actual_page = pager.get_next() + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page') # Assert page size self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') # Assert has_next False because n+1 limit is 1 more than user page size - self.assertFalse(pager.has_next(), 'has_next() should return False.') + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test correct items on page when n+1 def test_get_next_page_equal_to_limit(self): page_size = 14 mock = MockPageResponses(page_size+1, page_size) - with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPageIterator(self.client, {'limit': page_size}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): + page_iterator = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page - actual_page = pager.get_next() + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') # Assert page size self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') # Assert has_next True - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') # Get and assert second page - second_page = pager.get_next() + second_page = next(page_iterator) self.assertEqual(len(second_page), 1 , 'The second page should have one item.') # Note row keys are zero indexed so n+1 element that is first item on second page matches page size self.assertEqual(second_page[0].key, page_size, 'The first item key on the second page should match the page size number.') self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") - self.assertFalse(pager.has_next(), 'has_next() should return False.') + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test correct items on page when n+more def test_get_next_page_greater_than_limit(self): page_size = 7 mock = MockPageResponses(page_size+2, page_size) - with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPageIterator(self.client, {'limit': page_size}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): + page_iterator = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page - actual_page = pager.get_next() + actual_page = next(page_iterator) self.assertSequenceEqual(actual_page, mock.get_expected_page(1), 'The actual page should match the expected page.') # Assert page size self.assertEqual(len(actual_page), page_size, 'The actual page size should match the expected page size.') # Assert has_next True - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') # Get and assert second page - second_page = pager.get_next() + second_page = next(page_iterator) self.assertEqual(len(second_page), 2 , 'The second page should have two items.') # Note row keys are zero indexed so n+1 element that is first item on second page matches page size self.assertEqual(second_page[0].key, page_size, 'The first item key on the second page should match the page size number.') self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") - self.assertFalse(pager.has_next(), 'has_next() should return False.') + self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test getting all items def test_get_all(self): page_size = 3 mock = MockPageResponses(page_size*12, page_size) - with patch('test_pagination_key.KeyTestPager.operation', mock.get_next_page): - pager = KeyTestPageIterator(self.client, {'limit': page_size}) + pagination = Pagination(self.client, KeyTestPageIterator, {'limit': page_size}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): + pager = pagination.pager() # Get and assert all items self.assertSequenceEqual(pager.get_all(), mock.all_expected_items(), 'The results should match all the pages.') @@ -173,10 +174,10 @@ def test_no_boundary_check_by_default(self): ] expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) - with patch('test_pagination_key.KeyTestPager.operation', mockmock): - pager = KeyTestPageIterator(self.client, {'limit': 1}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mockmock): + page_iterator = KeyTestPageIterator(self.client, {'limit': 1}) # Get and assert page - self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) + self.assertSequenceEqual(next(page_iterator), (expected_rows[0],)) def test_boundary_failure_throws_on_get_next(self): mock_rows = [ @@ -185,17 +186,17 @@ def test_boundary_failure_throws_on_get_next(self): ] expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) - with patch('test_pagination_key.KeyTestPager.operation', mockmock): - pager = KeyTestPageIterator(self.client, {'limit': 1}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mockmock): + page_iterator = KeyTestPageIterator(self.client, {'limit': 1}) with patch( - 'test_pagination_key.KeyTestPager.boundary_func', + 'test_pagination_key.KeyTestPageIterator.boundary_func', lambda p,l: 'test error' if p.id == l.id and p.key == l.key else None): # Get and assert page - self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) + self.assertSequenceEqual(next(page_iterator), (expected_rows[0],)) # Assert has_next True - self.assertTrue(pager.has_next(), 'has_next() should return True.') + self.assertTrue(page_iterator._has_next, '_has_next should return True.') with self.assertRaises(Exception): - pager.get_next() + next(page_iterator) def test_no_boundary_check_when_no_items_left(self): mock_rows = [ @@ -203,12 +204,12 @@ def test_no_boundary_check_when_no_items_left(self): ] expected_rows = ViewResult.from_dict({'rows': mock_rows}).rows mockmock = Mock(return_value=DetailedResponse(response={'rows': mock_rows})) - with patch('test_pagination_key.KeyTestPager.operation', mockmock): - pager = KeyTestPageIterator(self.client, {'limit': 1}) + with patch('test_pagination_key.KeyTestPageIterator.operation', mockmock): + page_iterator = KeyTestPageIterator(self.client, {'limit': 1}) with patch( - 'test_pagination_key.KeyTestPager.boundary_func', + 'test_pagination_key.KeyTestPageIterator.boundary_func', Exception('Check boundary should not be called.')): # Get and assert page if boundary is checked, will raise exception - self.assertSequenceEqual(pager.get_next(), (expected_rows[0],)) + self.assertSequenceEqual(next(page_iterator), (expected_rows[0],)) # Assert has_next False - self.assertFalse(pager.has_next(), 'has_next() should return True.') + self.assertFalse(page_iterator._has_next, '_has_next should return True.') From b22b71a92a71772eeacfa3757dabc4de5ea41507 Mon Sep 17 00:00:00 2001 From: Richard Ellis Date: Fri, 4 Apr 2025 11:43:34 +0100 Subject: [PATCH 15/27] chore: remove TODO comment --- ibmcloudant/features/pagination.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 69806f9a..ff18c85f 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -163,7 +163,6 @@ def new_pagination(cls, client:CloudantV1, type: PagerType, **kwargs): if type == PagerType.POST_VIEW: return Pagination(client, _ViewPageIterator, kwargs) -# TODO state checks class _IteratorPagerState(Enum): NEW = auto() GET_NEXT = auto() From 06fdc02431470f4b1b0eee85d17579d5b6a06584 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 8 May 2025 13:48:15 +0100 Subject: [PATCH 16/27] feat: add pagination option validation --- ibmcloudant/features/pagination.py | 32 ++++++++++- .../test_pagination_option_validation.py | 57 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 test/unit/features/test_pagination_option_validation.py diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index ff18c85f..d2744e2a 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -22,7 +22,7 @@ """ from abc import abstractmethod -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator, Sequence from enum import auto, Enum from functools import partial from types import MappingProxyType @@ -39,6 +39,9 @@ # Type variable for the key in key based paging K = TypeVar('K') +_MAX_LIMIT = 200 +_MIN_LIMIT = 1 + class PagerType(Enum): """ Enumeration of the available Pager types @@ -135,6 +138,23 @@ def rows(self) -> Iterable[I]: for page in self.pages(): yield from page + @classmethod + def _validate_limit(cls, opts: dict): + limit: int | None = opts.get('limit') + # For None case the valid default limit of _MAX_LIMIT will be applied later + if limit is not None: + if limit > _MAX_LIMIT: + raise ValueError(f'The provided limit {limit} exceeds the maximum page size value of {_MAX_LIMIT}.') + if limit < _MIN_LIMIT: + raise ValueError(f'The provided limit {limit} is lower than the minimum page size value of {_MIN_LIMIT}.') + + @classmethod + def _validate_options_absent(cls, invalid_opts: Sequence[str], opts: dict): + # for each invalid_opts entry check if it is present in opts dict + for invalid_opt in invalid_opts: + if invalid_opt in opts: + raise ValueError(f"The option '{invalid_opt}' is invalid when using pagination.") + @classmethod def new_pagination(cls, client:CloudantV1, type: PagerType, **kwargs): """ @@ -144,23 +164,31 @@ def new_pagination(cls, client:CloudantV1, type: PagerType, **kwargs): kwargs: dict - the options for the operation """ + # Validate the limit + cls._validate_limit(kwargs) if type == PagerType.POST_ALL_DOCS: + cls._validate_options_absent(('keys',), kwargs) return Pagination(client, _AllDocsPageIterator, kwargs) if type == PagerType.POST_DESIGN_DOCS: + cls._validate_options_absent(('keys',), kwargs) return Pagination(client, _DesignDocsPageIterator, kwargs) if type == PagerType.POST_FIND: return Pagination(client, _FindPageIterator, kwargs) if type == PagerType.POST_PARTITION_ALL_DOCS: + cls._validate_options_absent(('keys',), kwargs) return Pagination(client, _AllDocsPartitionPageIterator, kwargs) if type == PagerType.POST_PARTITION_FIND: return Pagination(client, _FindPartitionPageIterator, kwargs) if type == PagerType.POST_PARTITION_SEARCH: return Pagination(client, _SearchPartitionPageIterator, kwargs) if type == PagerType.POST_PARTITION_VIEW: + cls._validate_options_absent(('keys',), kwargs) return Pagination(client, _ViewPartitionPageIterator, kwargs) if type == PagerType.POST_SEARCH: + cls._validate_options_absent(('counts', 'group_field', 'group_limit', 'group_sort', 'ranges',), kwargs) return Pagination(client, _SearchPageIterator, kwargs) if type == PagerType.POST_VIEW: + cls._validate_options_absent(('keys',), kwargs) return Pagination(client, _ViewPageIterator, kwargs) class _IteratorPagerState(Enum): @@ -261,7 +289,7 @@ def _next_request(self) -> list[I]: return items def _page_size_from_opts_limit(self, opts:dict) -> int: - return opts.get('limit', 200) + return opts.get('limit', _MAX_LIMIT) @abstractmethod def _result_converter(self) -> Callable[[dict], R]: diff --git a/test/unit/features/test_pagination_option_validation.py b/test/unit/features/test_pagination_option_validation.py new file mode 100644 index 00000000..c694262d --- /dev/null +++ b/test/unit/features/test_pagination_option_validation.py @@ -0,0 +1,57 @@ +# coding: utf-8 + +# © Copyright IBM Corporation 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase +from ibmcloudant.features.pagination import _MIN_LIMIT, _MAX_LIMIT, PagerType, Pagination + +class TestPaginationOptionValidation(TestCase): + + all_doc_paginations = (PagerType.POST_ALL_DOCS, PagerType.POST_PARTITION_ALL_DOCS, PagerType.POST_DESIGN_DOCS) + find_paginations = (PagerType.POST_PARTITION_FIND, PagerType.POST_FIND) + search_paginations = (PagerType.POST_SEARCH, PagerType.POST_PARTITION_SEARCH) + view_paginations = (PagerType.POST_PARTITION_VIEW, PagerType.POST_VIEW) + view_like_paginations = (*all_doc_paginations, *view_paginations) + all_paginations = (*find_paginations, *search_paginations, *view_like_paginations) + + def test_valid_limits(self): + test_limits = (_MIN_LIMIT, _MAX_LIMIT - 1, _MAX_LIMIT, None) + for limit in test_limits: + for pager_type in self.all_paginations: + with self.subTest(pager_type): + try: + Pagination.new_pagination(None, pager_type, limit=limit) + except ValueError: + self.fail('There should be no ValueError for valid limits.') + + def test_invalid_limits(self): + test_limits = (_MIN_LIMIT - 1, _MAX_LIMIT + 1) + for limit in test_limits: + for pager_type in self.all_paginations: + with self.subTest(pager_type): + with self.assertRaises(ValueError, msg='There should be a ValueError for invalid limits.'): + Pagination.new_pagination(None, pager_type, limit=limit) + + def test_keys_value_error_for_view_like(self): + for pager_type in self.view_like_paginations: + with self.subTest(pager_type): + with self.assertRaises(ValueError, msg=f'There should be a ValueError for {pager_type} with keys.'): + Pagination.new_pagination(None, pager_type, keys=['a','b','c']) + + def test_facet_value_errors_for_search(self): + for invalid_opt in ('counts', 'group_field', 'group_limit', 'group_sort', 'ranges',): + with self.subTest(invalid_opt): + with self.assertRaises(ValueError, msg=f'There should be a ValueError for search with option {invalid_opt}.'): + Pagination.new_pagination(None, PagerType.POST_SEARCH, **{invalid_opt: 'test value'}) From c100dc3d412424f2d3a52427081e2f3056bf1a8d Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 14 May 2025 16:26:10 +0100 Subject: [PATCH 17/27] fix: use Sequence type for interfaces --- ibmcloudant/features/pagination.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index d2744e2a..83a8b813 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -74,7 +74,7 @@ def has_next(self) -> bool: raise NotImplementedError() @abstractmethod - def get_next(self) -> tuple[I]: + def get_next(self) -> Sequence[I]: """ returns the next page of results """ @@ -82,7 +82,7 @@ def get_next(self) -> tuple[I]: raise NotImplementedError() @abstractmethod - def get_all(self) -> tuple[I]: + def get_all(self) -> Sequence[I]: """ returns all the pages of results in single list """ @@ -115,7 +115,7 @@ def pager(self) -> Pager[I]: return _IteratorPager(self.pages) - def pages(self) -> Iterable[tuple[I]]: + def pages(self) -> Iterable[Sequence[I]]: """ Create a new Iterable for all the pages. This type is useful for handling pages in a for loop. @@ -202,9 +202,9 @@ class _IteratorPager(Pager[I]): _state_mixed_msg = 'This pager has been consumed, use a new Pager.' _state_consumed_msg = 'Cannot mix get_all() and get_next() use only one method or make a new Pager.' - def __init__(self, iterable_func: Callable[[], Iterator[tuple[I]]]): - self._iterable_func: Callable[[], Iterator[tuple[I]]] = iterable_func - self._iterator: Iterator[tuple[I]] = iter(self._iterable_func()) + def __init__(self, iterable_func: Callable[[], Iterator[Sequence[I]]]): + self._iterable_func: Callable[[], Iterator[Sequence[I]]] = iterable_func + self._iterator: Iterator[Sequence[I]] = iter(self._iterable_func()) self._state: _IteratorPagerState = _IteratorPagerState.NEW def has_next(self) -> bool: @@ -214,17 +214,17 @@ def has_next(self) -> bool: return self._iterator._has_next - def get_next(self) -> tuple[I]: + def get_next(self) -> Sequence[I]: """ returns the next page of results """ self._check_state(mode=_IteratorPagerState.GET_NEXT) - page: tuple[I] = next(self._iterator) + page: Sequence[I] = next(self._iterator) if not self._iterator._has_next: self._state = _IteratorPagerState.CONSUMED return page - def get_all(self) -> tuple[I]: + def get_all(self) -> Sequence[I]: """ returns all the pages of results in single list """ @@ -246,7 +246,7 @@ def _check_state(self, mode: _IteratorPagerState): raise Exception(_IteratorPager._state_consumed_msg) raise Exception(_IteratorPager._state_mixed_msg) -class _BasePageIterator(Iterator[tuple[I]]): +class _BasePageIterator(Iterator[Sequence[I]]): def __init__(self, client: CloudantV1, @@ -269,10 +269,10 @@ def __init__(self, # Partial method with the fixed ops self._next_request_function: Callable[..., DetailedResponse] = partial(operation, **fixed_opts) - def __iter__(self) -> Iterator[tuple[I]]: + def __iter__(self) -> Iterator[Sequence[I]]: return self - def __next__(self) -> tuple[I]: + def __next__(self) -> Sequence[I]: if self._has_next: return (*self._next_request(),) raise StopIteration() From 6c61be8ff5fd27727b8345486f358f278e3b3f96 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 14 May 2025 16:26:36 +0100 Subject: [PATCH 18/27] fix: add missing type --- ibmcloudant/features/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 83a8b813..daa1172a 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -102,7 +102,7 @@ class Pagination: * :meth:`rows` - for a row Iterable """ - def __init__(self, client: CloudantV1, type, opts: dict): + def __init__(self, client: CloudantV1, type: PagerType, opts: dict): self._client = client self._operation_type = type self._initial_opts = dict(opts) From 437941f18afeea0aa07a1ac3d3f921f8a8bb5029 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 14 May 2025 16:29:14 +0100 Subject: [PATCH 19/27] fix: opts deletion error --- ibmcloudant/features/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index daa1172a..74bfeed9 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -355,7 +355,7 @@ def _result_converter(self) -> Callable[[dict], AllDocsResult]: def _get_next_page_options(self, result: R) -> dict: # Remove start_key_doc_id for all_docs paging opts: dict = super()._get_next_page_options(result) - del opts.start_key_doc_id + del opts['start_key_doc_id'] def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: # IDs are always unique in _all_docs pagers so return None From 7ec0d2d0442c3e50ebfbfa9ec75299a852479818 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 14 May 2025 17:12:50 +0100 Subject: [PATCH 20/27] fix: return from all docs _get_next_page_options --- ibmcloudant/features/pagination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index 74bfeed9..db62f3ac 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -356,6 +356,7 @@ def _get_next_page_options(self, result: R) -> dict: # Remove start_key_doc_id for all_docs paging opts: dict = super()._get_next_page_options(result) del opts['start_key_doc_id'] + return opts def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: # IDs are always unique in _all_docs pagers so return None From d838902412fb654f85114a804f6c6a6ec704156e Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 15 May 2025 11:24:29 +0100 Subject: [PATCH 21/27] test: extract common page mocks --- test/unit/features/conftest.py | 129 ++++++++++++++++++ test/unit/features/test_pagination_base.py | 87 +++++------- .../unit/features/test_pagination_bookmark.py | 50 ++----- test/unit/features/test_pagination_key.py | 50 ++----- 4 files changed, 179 insertions(+), 137 deletions(-) diff --git a/test/unit/features/conftest.py b/test/unit/features/conftest.py index e4e373e7..2566b22f 100644 --- a/test/unit/features/conftest.py +++ b/test/unit/features/conftest.py @@ -18,6 +18,7 @@ Shared tests' fixtures """ +from typing import Sequence import unittest import pytest import os @@ -31,11 +32,17 @@ from requests import codes from requests.exceptions import ConnectionError +from ibm_cloud_sdk_core import DetailedResponse + from ibmcloudant.cloudant_v1 import ( + AllDocsResult, CloudantV1, + FindResult, PostChangesEnums, ChangesResult, ChangesResultItem, + SearchResult, + ViewResult, ) from ibmcloudant.features.changes_follower import ( @@ -43,6 +50,7 @@ _BATCH_SIZE, _Mode, ) +from ibmcloudant.features.pagination import PagerType @pytest.fixture(scope='class') @@ -241,3 +249,124 @@ def main(): return counter return main() + +class PaginationMockSupport: + all_docs_pagers: Sequence[PagerType] = ( + PagerType.POST_ALL_DOCS, + PagerType.POST_DESIGN_DOCS, + PagerType.POST_PARTITION_ALL_DOCS + ) + view_pagers: Sequence[PagerType] = ( + PagerType.POST_VIEW, + PagerType.POST_PARTITION_VIEW + ) + # the key pager types (n+1 paging) + key_pagers: Sequence[PagerType] = all_docs_pagers + view_pagers + find_pagers: Sequence[PagerType] = ( + PagerType.POST_FIND, + PagerType.POST_PARTITION_FIND) + search_pagers: Sequence[PagerType] = ( + PagerType.POST_SEARCH, + PagerType.POST_PARTITION_SEARCH) + + # Map of pager type to a tuple of patch function name, result wrapper lambda, result row lambda + operation_map: dict[PagerType:str] = { + PagerType.POST_ALL_DOCS: 'ibmcloudant.cloudant_v1.CloudantV1.post_all_docs', + PagerType.POST_DESIGN_DOCS: 'ibmcloudant.cloudant_v1.CloudantV1.post_design_docs', + PagerType.POST_FIND: 'ibmcloudant.cloudant_v1.CloudantV1.post_find', + PagerType.POST_PARTITION_ALL_DOCS: 'ibmcloudant.cloudant_v1.CloudantV1.post_partition_all_docs', + PagerType.POST_PARTITION_FIND: 'ibmcloudant.cloudant_v1.CloudantV1.post_partition_find', + PagerType.POST_PARTITION_SEARCH: 'ibmcloudant.cloudant_v1.CloudantV1.post_partition_search', + PagerType.POST_PARTITION_VIEW: 'ibmcloudant.cloudant_v1.CloudantV1.post_partition_view', + PagerType.POST_SEARCH: 'ibmcloudant.cloudant_v1.CloudantV1.post_search', + PagerType.POST_VIEW: 'ibmcloudant.cloudant_v1.CloudantV1.post_view' + } + + def make_wrapper(pager_type: PagerType, total: int, rows: Sequence) -> dict[str:any]: + if pager_type in PaginationMockSupport.key_pagers: + return {'total_rows': total, 'rows': rows} + else: + bkmk = 'emptypagebookmark' + last_row = rows[-1] if len(rows) > 0 else None + if pager_type in PaginationMockSupport.find_pagers: + return {'bookmark': last_row['_id'] if last_row else bkmk, 'docs': rows} + elif pager_type in PaginationMockSupport.search_pagers: + return {'bookmark': last_row['id'] if last_row else bkmk, 'total_rows': total, 'rows': rows} + else: + raise Exception('Unknown pager type, fail test.') + + def make_row(pager_type: PagerType, i: int) -> dict[str:any]: + id = f'testdoc{i}' + rev = f'1-abc{i}' + if pager_type in PaginationMockSupport.key_pagers: + if pager_type in (PagerType.POST_VIEW, PagerType.POST_PARTITION_VIEW): + key = i + value = 1 + else: + key = id + value = {'rev': rev} + return {'id': id, 'key': key, 'value': value} + elif pager_type in PaginationMockSupport.find_pagers: + return {'_id':id, '_rev': rev, 'testfield': i} + elif pager_type in PaginationMockSupport.search_pagers: + return {'fields':{}, 'id': id} + else: + raise Exception('Unknown pager type, fail test.') + +class PaginationMockResponse: + """ + Test class for mocking page responses. + """ + def __init__(self, + total_items: int, + page_size: int, + pager_type: PagerType + ): + self.total_items: int = total_items + self.page_size: int = page_size + self.pages = self.generator() + self.pager_type: PagerType = pager_type + self.plus_one_paging: bool = self.pager_type in PaginationMockSupport.key_pagers + self.expected_pages: list[list] = [] + + def generator(self): + for page in itertools.batched(range(0, self.total_items), self.page_size): + rows = [PaginationMockSupport.make_row(self.pager_type, i) for i in page] + if self.plus_one_paging: + # Add an n+1 row for key based paging if more pages + if (n_plus_one := page[-1] + 1) < self.total_items: + rows.append(PaginationMockSupport.make_row(self.pager_type, n_plus_one)) + yield DetailedResponse(response=PaginationMockSupport.make_wrapper(self.pager_type, self.total_items, rows)) + yield DetailedResponse(response=PaginationMockSupport.make_wrapper(self.pager_type, self.total_items, [])) + + def convert_result(self, result: dict) -> Sequence: + if self.pager_type in PaginationMockSupport.all_docs_pagers: + return AllDocsResult.from_dict(result).rows + elif self.pager_type in PaginationMockSupport.find_pagers: + return FindResult.from_dict(result).docs + elif self.pager_type in PaginationMockSupport.search_pagers: + return SearchResult.from_dict(result).rows + elif self.pager_type in PaginationMockSupport.view_pagers: + return ViewResult.from_dict(result).rows + + def get_next_page(self, **kwargs): + # return next(self.pages) + # ignore kwargs + # get next page + page = next(self.pages) + # convert to an expected page + rows = self.convert_result(page.get_result()) + if len(rows) > self.page_size and self.plus_one_paging: + self.expected_pages.append(rows[:-1]) + else: + self.expected_pages.append(rows) + return page + + def get_expected_page(self, page: int) -> list: + return self.expected_pages[page - 1] + + def all_expected_items(self) -> list: + all_items: list = [] + for page in self.expected_pages: + all_items.extend(page) + return all_items diff --git a/test/unit/features/test_pagination_base.py b/test/unit/features/test_pagination_base.py index 2071fd76..9a3caa29 100644 --- a/test/unit/features/test_pagination_base.py +++ b/test/unit/features/test_pagination_base.py @@ -15,13 +15,21 @@ # limitations under the License. from collections.abc import Callable -from itertools import batched from typing import Iterable from unittest.mock import Mock, patch -from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow -from ibmcloudant.features.pagination import _BasePageIterator, _IteratorPager, Pager, Pagination -from conftest import MockClientBaseCase +from ibmcloudant.features.pagination import _BasePageIterator, _IteratorPager, Pager, PagerType, Pagination +from conftest import MockClientBaseCase, PaginationMockResponse + +class BasePageMockResponses(PaginationMockResponse): + """ + Test class for mocking page responses. + """ + def __init__(self, total_items: int, page_size: int): + super().__init__(total_items, page_size, PagerType.POST_VIEW) + # This test uses View structures, but doesn't do n+1 like a view/key pager + # Override plus_one_paging to accommodate this weird hybrid + self.plus_one_paging = False class TestPageIterator(_BasePageIterator): """ @@ -45,39 +53,6 @@ def _get_next_page_options(self, result: ViewResult) -> dict: else: return {'start_key': result.rows[-1].key} -class MockPageResponses: - """ - Test class for mocking page responses. - """ - def __init__(self, total_items: int, page_size: int): - self.total_items: int = total_items - self.page_size: int = page_size - self.pages = self.generator() - self.expected_pages: list[list[ViewResultRow]] = [] - - def generator(self): - for page in batched(range(0, self.total_items), self.page_size): - rows = [{'id':str(i), 'key':i, 'value':i} for i in page] - yield DetailedResponse(response={'rows': rows}) - yield DetailedResponse(response={'rows': []}) - - def get_next_page(self, **kwargs): - # ignore kwargs - # get next page - page = next(self.pages) - # convert to an expected page - self.expected_pages.append(ViewResult.from_dict(page.get_result()).rows) - return page - - def get_expected_page(self, page: int) -> list[ViewResultRow]: - return self.expected_pages[page - 1] - - def all_expected_items(self) -> list[ViewResultRow]: - all_items: list[ViewResultRow] = [] - for page in self.expected_pages: - all_items.extend(page) - return all_items - class TestBasePageIterator(MockClientBaseCase): def test_init(self): operation = self.client.post_view @@ -128,7 +103,7 @@ def test_has_next_initially_true(self): def test_has_next_true_for_result_equal_to_limit(self): page_size = 1 # Init with mock that returns only a single row - with patch('test_pagination_base.TestPageIterator.operation', MockPageResponses(1, page_size).get_next_page): + with patch('test_pagination_base.TestPageIterator.operation', BasePageMockResponses(1, page_size).get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) @@ -140,7 +115,7 @@ def test_has_next_true_for_result_equal_to_limit(self): def test_has_next_false_for_result_less_than_limit(self): page_size = 1 # Init with mock that returns zero rows - with patch('test_pagination_base.TestPageIterator.operation', MockPageResponses(0, page_size).get_next_page): + with patch('test_pagination_base.TestPageIterator.operation', BasePageMockResponses(0, page_size).get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, {'limit': page_size}) @@ -152,7 +127,7 @@ def test_has_next_false_for_result_less_than_limit(self): def test_next_first_page(self): page_size = 25 # Mock that returns one page of 25 items - mock = MockPageResponses(page_size, page_size) + mock = BasePageMockResponses(page_size, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -165,7 +140,7 @@ def test_next_first_page(self): def test_next_two_pages(self): page_size = 3 # Mock that returns two pages of 3 items - mock = MockPageResponses(2*page_size, page_size) + mock = BasePageMockResponses(2*page_size, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -186,7 +161,7 @@ def test_next_two_pages(self): def test_next_until_empty(self): page_size = 3 # Mock that returns 3 pages of 3 items - mock = MockPageResponses(3*page_size, page_size) + mock = BasePageMockResponses(3*page_size, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -205,7 +180,7 @@ def test_next_until_empty(self): def test_next_until_smaller(self): page_size = 3 # Mock that returns 3 pages of 3 items, then 1 more page with 1 item - mock = MockPageResponses(3*page_size + 1, page_size) + mock = BasePageMockResponses(3*page_size + 1, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -224,7 +199,7 @@ def test_next_until_smaller(self): def test_next_exception(self): page_size = 2 # Mock that returns one page of one item - mock = MockPageResponses(page_size - 1, page_size) + mock = BasePageMockResponses(page_size - 1, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -241,7 +216,7 @@ def test_next_exception(self): def test_pages_immutable(self): page_size = 1 - mock = MockPageResponses(page_size, page_size) + mock = BasePageMockResponses(page_size, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -253,7 +228,7 @@ def test_pages_immutable(self): def test_set_next_page_options(self): page_size = 1 - mock = MockPageResponses(5*page_size, page_size) + mock = BasePageMockResponses(5*page_size, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -271,7 +246,7 @@ def test_set_next_page_options(self): def test_next_resumes_after_error(self): page_size = 1 - mock = MockPageResponses(3*page_size, page_size) + mock = BasePageMockResponses(3*page_size, page_size) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): page_iterator: Iterable[ViewResultRow] = TestPageIterator( self.client, @@ -292,7 +267,7 @@ def test_next_resumes_after_error(self): def test_pages_iterable(self): page_size = 23 - mock = MockPageResponses(3*page_size-1, page_size) + mock = BasePageMockResponses(3*page_size-1, page_size) pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): # Check pages are iterable @@ -305,7 +280,7 @@ def test_pages_iterable(self): def test_rows_iterable(self): page_size = 23 - mock = MockPageResponses(3*page_size-1, page_size) + mock = BasePageMockResponses(3*page_size-1, page_size) pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): actual_items = [] @@ -317,7 +292,7 @@ def test_rows_iterable(self): def test_as_pager_get_next_first_page(self): page_size = 7 # Mock that returns two pages of 7 items - mock = MockPageResponses(2*page_size, page_size) + mock = BasePageMockResponses(2*page_size, page_size) pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): pager = pagination.pager() @@ -328,7 +303,7 @@ def test_as_pager_get_next_first_page(self): def test_as_pager_get_all(self): page_size = 11 # Mock that returns 6 pages of 11 items, then 1 more page with 5 items - mock = MockPageResponses(71, page_size) + mock = BasePageMockResponses(71, page_size) pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): pager: Pager[ViewResultRow] = pagination.pager() @@ -340,7 +315,7 @@ def test_as_pager_get_all(self): def test_as_pager_get_all_restarts_after_error(self): page_size = 1 - mock = MockPageResponses(2*page_size, page_size) + mock = BasePageMockResponses(2*page_size, page_size) first_page = mock.get_next_page() # mock response order # first page, error, first page replay, second page @@ -360,7 +335,7 @@ def test_as_pager_get_all_restarts_after_error(self): def test_as_pager_get_next_get_all_throws(self): page_size = 11 # Mock that returns 6 pages of 11 items, then 1 more page with 5 items - mock = MockPageResponses(71, page_size) + mock = BasePageMockResponses(71, page_size) pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): pager: Pager[ViewResultRow] = pagination.pager() @@ -374,7 +349,7 @@ def test_as_pager_get_next_get_all_throws(self): def test_as_pager_get_all_get_next_throws(self): page_size = 1 - mock = MockPageResponses(2*page_size, page_size) + mock = BasePageMockResponses(2*page_size, page_size) first_page = mock.get_next_page() # mock response order # first page, error, first page replay, second page @@ -394,7 +369,7 @@ def test_as_pager_get_all_get_next_throws(self): def test_as_pager_get_next_resumes_after_error(self): page_size = 1 - mock = MockPageResponses(2*page_size, page_size) + mock = BasePageMockResponses(2*page_size, page_size) # mock response order # first page, error, second page mockmock = Mock(side_effect=[ @@ -415,7 +390,7 @@ def test_as_pager_get_next_resumes_after_error(self): def test_as_pager_get_next_until_consumed(self): page_size = 7 # Mock that returns two pages of 7 items - mock = MockPageResponses(2*page_size, page_size) + mock = BasePageMockResponses(2*page_size, page_size) pagination = Pagination(self.client, TestPageIterator, {'limit': page_size}) with patch('test_pagination_base.TestPageIterator.operation', mock.get_next_page): pager = pagination.pager() diff --git a/test/unit/features/test_pagination_bookmark.py b/test/unit/features/test_pagination_bookmark.py index a28de666..7ad4c952 100644 --- a/test/unit/features/test_pagination_bookmark.py +++ b/test/unit/features/test_pagination_bookmark.py @@ -15,12 +15,10 @@ # limitations under the License. from collections.abc import Callable, Iterator -from itertools import batched -from unittest.mock import Mock, patch -from ibm_cloud_sdk_core import DetailedResponse +from unittest.mock import patch from ibmcloudant.cloudant_v1 import SearchResult, SearchResultRow -from ibmcloudant.features.pagination import _BookmarkPageIterator, Pager, Pagination -from conftest import MockClientBaseCase +from ibmcloudant.features.pagination import _BookmarkPageIterator, PagerType, Pagination +from conftest import MockClientBaseCase, PaginationMockResponse class BookmarkTestPageIterator(_BookmarkPageIterator): """ @@ -38,38 +36,12 @@ def _result_converter(self) -> Callable[[dict], SearchResult]: def _items(self, result: SearchResult) -> tuple[SearchResultRow]: return result.rows -class MockPageResponses: +class BookmarkPaginationMockResponses(PaginationMockResponse): """ Test class for mocking page responses. """ def __init__(self, total_items: int, page_size: int): - self.total_items: int = total_items - self.page_size: int = page_size - self.pages = self.generator() - self.expected_pages: list[list[SearchResultRow]] = [] - - def generator(self): - for page in batched(range(0, self.total_items), self.page_size): - rows = [{'id':str(i), 'fields': {'value': i}} for i in page] - yield DetailedResponse(response={'total_rows': self.total_items, 'bookmark': rows[-1]['id'], 'rows': rows}) - yield DetailedResponse(response={'total_rows': self.total_items, 'bookmark': 'last', 'rows': []}) - - def get_next_page(self, **kwargs): - # ignore kwargs - # get next page - page = next(self.pages) - # convert to an expected page - self.expected_pages.append(SearchResult.from_dict(page.get_result()).rows) - return page - - def get_expected_page(self, page: int) -> list[SearchResultRow]: - return self.expected_pages[page - 1] - - def all_expected_items(self) -> list[SearchResultRow]: - all_items: list[SearchResultRow] = [] - for page in self.expected_pages: - all_items.extend(page) - return all_items + super().__init__(total_items, page_size, PagerType.POST_SEARCH) class TestBookmarkPageIterator(MockClientBaseCase): @@ -88,7 +60,7 @@ def test_limit_page_size(self): # Test all items on page when no more pages def test_get_next_page_less_than_limit(self): page_size = 21 - mock = MockPageResponses(page_size - 1, page_size) + mock = BookmarkPaginationMockResponses(page_size - 1, page_size) with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): page_iterator = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page @@ -102,7 +74,7 @@ def test_get_next_page_less_than_limit(self): # Test correct items on page when limit def test_get_next_page_equal_to_limit(self): page_size = 14 - mock = MockPageResponses(page_size, page_size) + mock = BookmarkPaginationMockResponses(page_size, page_size) with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): page_iterator = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page @@ -113,7 +85,7 @@ def test_get_next_page_equal_to_limit(self): # Assert has_next True self.assertTrue(page_iterator._has_next, '_has_next should return True.') # Assert bookmark - self.assertEqual(page_iterator._next_page_opts['bookmark'], str(page_size - 1), 'The bookmark should be one less than the page size.') + self.assertEqual(page_iterator._next_page_opts['bookmark'], f'testdoc{page_size - 1}', 'The bookmark should be one less than the page size.') # Get and assert second page second_page = next(page_iterator) # Note row keys are zero indexed so page size - 1 @@ -123,7 +95,7 @@ def test_get_next_page_equal_to_limit(self): # Test correct items on page when n+more def test_get_next_page_greater_than_limit(self): page_size = 7 - mock = MockPageResponses(page_size+2, page_size) + mock = BookmarkPaginationMockResponses(page_size+2, page_size) with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): page_iterator = BookmarkTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page @@ -137,14 +109,14 @@ def test_get_next_page_greater_than_limit(self): second_page = next(page_iterator) self.assertEqual(len(second_page), 2 , 'The second page should have two items.') # Note row keys are zero indexed so n+1 element that is first item on second page matches page size - self.assertEqual(second_page[0].id, str(page_size), 'The first item key on the second page should match the page size number.') + self.assertEqual(second_page[0].id, f'testdoc{page_size}', 'The first item key on the second page should match the page size number.') self.assertSequenceEqual(second_page, mock.get_expected_page(2), "The actual page should match the expected page") self.assertFalse(page_iterator._has_next, '_has_next should return False.') # Test getting all items def test_get_all(self): page_size = 3 - mock = MockPageResponses(page_size*12, page_size) + mock = BookmarkPaginationMockResponses(page_size*12, page_size) pagination = Pagination(self.client, BookmarkTestPageIterator, {'limit': page_size}) with patch('test_pagination_bookmark.BookmarkTestPageIterator.operation', mock.get_next_page): pager = pagination.pager() diff --git a/test/unit/features/test_pagination_key.py b/test/unit/features/test_pagination_key.py index c9a16215..16faf35f 100644 --- a/test/unit/features/test_pagination_key.py +++ b/test/unit/features/test_pagination_key.py @@ -15,12 +15,11 @@ # limitations under the License. from collections.abc import Callable, Iterator -from itertools import batched from unittest.mock import Mock, patch from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import ViewResult, ViewResultRow -from ibmcloudant.features.pagination import _KeyPageIterator, Pager, Pagination -from conftest import MockClientBaseCase +from ibmcloudant.features.pagination import _KeyPageIterator, PagerType, Pagination +from conftest import MockClientBaseCase, PaginationMockResponse class KeyTestPageIterator(_KeyPageIterator): """ @@ -47,45 +46,12 @@ def _get_next_page_options(self, result: ViewResult) -> dict: def check_boundary(self, penultimate_item, last_item): return KeyTestPageIterator.boundary_func(penultimate_item, last_item) -class MockPageResponses: +class KeyPaginationMockResponses(PaginationMockResponse): """ Test class for mocking page responses. """ def __init__(self, total_items: int, page_size: int): - self.total_items: int = total_items - self.page_size: int = page_size - self.pages = self.generator() - self.expected_pages: list[list[ViewResultRow]] = [] - - def generator(self): - for page in batched(range(0, self.total_items), self.page_size): - rows = [{'id':str(i), 'key':i, 'value':i} for i in page] - # Add an n+1 row for key based paging if more pages - if (n_plus_one := page[-1] + 1) < self.total_items: - rows.append({'id':str(n_plus_one), 'key':n_plus_one, 'value':n_plus_one}) - yield DetailedResponse(response={'rows': rows}) - yield DetailedResponse(response={'rows': []}) - - def get_next_page(self, **kwargs): - # ignore kwargs - # get next page - page = next(self.pages) - # convert to an expected page, removing the n+1 row if needed - result = ViewResult.from_dict(page.get_result()) - if len(result.rows) > self.page_size: - self.expected_pages.append(result.rows[:-1]) - else: - self.expected_pages.append(result.rows) - return page - - def get_expected_page(self, page: int) -> list[ViewResultRow]: - return self.expected_pages[page - 1] - - def all_expected_items(self) -> list[ViewResultRow]: - all_items: list[ViewResultRow] = [] - for page in self.expected_pages: - all_items.extend(page) - return all_items + super().__init__(total_items, page_size, PagerType.POST_VIEW) class TestKeyPageIterator(MockClientBaseCase): @@ -104,7 +70,7 @@ def test_limit_page_size(self): # Test all items on page when no more pages def test_get_next_page_less_than_limit(self): page_size = 21 - mock = MockPageResponses(page_size, page_size) + mock = KeyPaginationMockResponses(page_size, page_size) with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): page_iterator = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page @@ -118,7 +84,7 @@ def test_get_next_page_less_than_limit(self): # Test correct items on page when n+1 def test_get_next_page_equal_to_limit(self): page_size = 14 - mock = MockPageResponses(page_size+1, page_size) + mock = KeyPaginationMockResponses(page_size+1, page_size) with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): page_iterator = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page @@ -139,7 +105,7 @@ def test_get_next_page_equal_to_limit(self): # Test correct items on page when n+more def test_get_next_page_greater_than_limit(self): page_size = 7 - mock = MockPageResponses(page_size+2, page_size) + mock = KeyPaginationMockResponses(page_size+2, page_size) with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): page_iterator = KeyTestPageIterator(self.client, {'limit': page_size}) # Get and assert first page @@ -160,7 +126,7 @@ def test_get_next_page_greater_than_limit(self): # Test getting all items def test_get_all(self): page_size = 3 - mock = MockPageResponses(page_size*12, page_size) + mock = KeyPaginationMockResponses(page_size*12, page_size) pagination = Pagination(self.client, KeyTestPageIterator, {'limit': page_size}) with patch('test_pagination_key.KeyTestPageIterator.operation', mock.get_next_page): pager = pagination.pager() From ab5139797c642d06f9e7aa75f3a464f282297538 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 15 May 2025 09:21:33 +0100 Subject: [PATCH 22/27] test: add pagination operation tests --- .../features/test_pagination_operations.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 test/unit/features/test_pagination_operations.py diff --git a/test/unit/features/test_pagination_operations.py b/test/unit/features/test_pagination_operations.py new file mode 100644 index 00000000..0b5d865d --- /dev/null +++ b/test/unit/features/test_pagination_operations.py @@ -0,0 +1,118 @@ +# coding: utf-8 + +# © Copyright IBM Corporation 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch +from conftest import MockClientBaseCase, PaginationMockSupport, PaginationMockResponse +from ibmcloudant.features.pagination import Pager, PagerType, Pagination + +class TestPaginationOperations(MockClientBaseCase): + + test_page_size = 10 + # tuples of test parameters + # total items, page size + page_sets = ( + (0, test_page_size), + (1, test_page_size), + (test_page_size - 1, test_page_size), + (test_page_size, test_page_size), + (test_page_size + 1, test_page_size), + (3 * test_page_size, test_page_size), + (3 * test_page_size + 1, test_page_size), + (4 * test_page_size - 1, test_page_size), + ) + + def get_expected_pages(self, total: int, page_size: int, pager_type: PagerType) -> int: + full_pages = total // page_size + partial_pages = 0 if (total % page_size == 0) else 1 + expected_pages = full_pages + partial_pages + # Need at least 1 empty page to know there are no more results + # if not ending on a partial page, except if the first page or + # using n+1 paging (because an exact user page is a partial real page). + if partial_pages == 0 and (expected_pages == 0 or pager_type not in PaginationMockSupport.key_pagers): + expected_pages += 1; # Will get at least 1 empty page + return expected_pages + + def test_pager(self): + for pager_type in PagerType: + with self.subTest(pager_type): + for page_set in self.page_sets: + with self.subTest(page_set): + actual_items: set = set() + actual_item_count: int = 0 + actual_page_count: int = 0 + expected_items_count: int = page_set[0] + page_size: int = page_set[1] + expected_page_count: int = self.get_expected_pages(expected_items_count, page_size, pager_type) + with patch(PaginationMockSupport.operation_map[pager_type], PaginationMockResponse(expected_items_count, page_size, pager_type).get_next_page): + pager: Pager = Pagination.new_pagination(self.client, pager_type, limit=page_size).pager() + while(pager.has_next()): + page = pager.get_next() + actual_page_count += 1 + actual_item_count += len(page) + for row in page: + if pager_type in (PagerType.POST_FIND, PagerType.POST_PARTITION_FIND): + id = row._id + else: + id = row.id + actual_items.add(id) + self.assertEqual(actual_page_count, expected_page_count, 'There should be the expected number of pages.') + self.assertEqual(actual_item_count, expected_items_count, 'There should be the expected number of items.') + self.assertEqual(len(actual_items), expected_items_count, 'The items should be unique.') + + def test_pages(self): + for pager_type in PagerType: + with self.subTest(pager_type): + for page_set in self.page_sets: + with self.subTest(page_set): + actual_items: set = set() + actual_item_count: int = 0 + actual_page_count: int = 0 + expected_items_count: int = page_set[0] + page_size: int = page_set[1] + expected_page_count: int = self.get_expected_pages(expected_items_count, page_size, pager_type) + with patch(PaginationMockSupport.operation_map[pager_type], PaginationMockResponse(expected_items_count, page_size, pager_type).get_next_page): + for page in Pagination.new_pagination(self.client, pager_type, limit=page_size).pages(): + actual_page_count += 1 + actual_item_count += len(page) + for row in page: + if pager_type in (PagerType.POST_FIND, PagerType.POST_PARTITION_FIND): + id = row._id + else: + id = row.id + actual_items.add(id) + self.assertEqual(actual_page_count, expected_page_count, 'There should be the expected number of pages.') + self.assertEqual(actual_item_count, expected_items_count, 'There should be the expected number of items.') + self.assertEqual(len(actual_items), expected_items_count, 'The items should be unique.') + + def test_rows(self): + for pager_type in PagerType: + with self.subTest(pager_type): + for page_set in self.page_sets: + with self.subTest(page_set): + actual_items: set = set() + actual_item_count: int = 0 + expected_items_count: int = page_set[0] + page_size: int = page_set[1] + with patch(PaginationMockSupport.operation_map[pager_type], PaginationMockResponse(expected_items_count, page_size, pager_type).get_next_page): + for row in Pagination.new_pagination(self.client, pager_type, limit=page_size).rows(): + actual_item_count += 1 + if pager_type in (PagerType.POST_FIND, PagerType.POST_PARTITION_FIND): + id = row._id + else: + id = row.id + actual_items.add(id) + self.assertEqual(actual_item_count, expected_items_count, 'There should be the expected number of items.') + self.assertEqual(len(actual_items), expected_items_count, 'The items should be unique.') From 98a7688c0ec6df750d184335390529f108c4fbc6 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 May 2025 17:44:28 +0100 Subject: [PATCH 23/27] test: share error code --- test/unit/features/conftest.py | 62 +++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/test/unit/features/conftest.py b/test/unit/features/conftest.py index 2566b22f..a76351fc 100644 --- a/test/unit/features/conftest.py +++ b/test/unit/features/conftest.py @@ -32,7 +32,7 @@ from requests import codes from requests.exceptions import ConnectionError -from ibm_cloud_sdk_core import DetailedResponse +from ibm_cloud_sdk_core import ApiException, DetailedResponse from ibmcloudant.cloudant_v1 import ( AllDocsResult, @@ -124,6 +124,38 @@ def setUpClass(cls): service_name='TEST_SERVICE', ) + def make_error_tuple(self, error: str) -> tuple[any]: + if error == 'bad_io': + return (200, {}, ConnectionError('peer reset')) + elif error == 'bad_json': + return (200, {}, '{') + else: + return ( + codes[error], + {}, + json.dumps({'error': error}), + ) + + def make_error(self, error: str) -> dict[str:any]: + error_tuple = self.make_error_tuple(error) + if error == 'bad_io': + return {'body': error_tuple[2]} + else: + return { + 'status': error_tuple[0], + 'content_type': 'application/json', + 'body': error_tuple[2], + } + + def make_error_exception(self, error: str) -> Exception: + error_dict = self.make_error(error) + if error == 'bad_io': + return error_dict['body'] + elif error == 'bad_json': + return ApiException(code=error_dict['status'], + message='Error processing the HTTP response',) + return ApiException(error_dict['status']) + class ChangesFollowerBaseCase(MockClientBaseCase): def prepare_mock_changes( @@ -143,16 +175,7 @@ def __call__(self, request): if self._return_error: self._return_error = False error = next(self._errors) - if error == 'bad_io': - return (200, {}, ConnectionError('peer reset')) - elif error == 'bad_json': - return (200, {}, '{') - else: - return ( - codes[error], - {}, - json.dumps({'error': error}), - ) + return self.make_error_tuple(error) # this stands for "large" seq in empty result case last_seq = f'{batches * _BATCH_SIZE}-abcdef' pending = 0 @@ -208,22 +231,7 @@ def __call__(self, request): def prepare_mock_with_error(self, error: str): _base_url = os.environ.get('TEST_SERVER_URL', 'http://localhost:5984') url = _base_url + '/db/_changes' - if error == 'bad_io': - return responses.post(url, body=ConnectionError('peer reset')) - elif error == 'bad_json': - return responses.post( - url, - status=200, - content_type='application/json', - body='{', - ) - else: - return responses.post( - url, - status=codes[error], - content_type='application/json', - json={'error': error}, - ) + return responses.post(url, **self.make_error(error)) def runner(self, follower, mode, timeout=1, stop_after=0): """ From be20499168f6ae7c92b54ce15a53ac8ff429d6d9 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 May 2025 17:44:41 +0100 Subject: [PATCH 24/27] test: add pagination error tests --- .../features/test_pagination_operations.py | 77 ++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/test/unit/features/test_pagination_operations.py b/test/unit/features/test_pagination_operations.py index 0b5d865d..16383431 100644 --- a/test/unit/features/test_pagination_operations.py +++ b/test/unit/features/test_pagination_operations.py @@ -15,9 +15,12 @@ # limitations under the License. from unittest.mock import patch + +import pytest from conftest import MockClientBaseCase, PaginationMockSupport, PaginationMockResponse -from ibmcloudant.features.pagination import Pager, PagerType, Pagination +from ibmcloudant.features.pagination import _MAX_LIMIT, Pager, PagerType, Pagination +@pytest.mark.usefixtures("errors") class TestPaginationOperations(MockClientBaseCase): test_page_size = 10 @@ -116,3 +119,75 @@ def test_rows(self): actual_items.add(id) self.assertEqual(actual_item_count, expected_items_count, 'There should be the expected number of items.') self.assertEqual(len(actual_items), expected_items_count, 'The items should be unique.') + + def test_pager_errors(self): + page_size = _MAX_LIMIT + for pager_type in PagerType: + with self.subTest(pager_type): + for error in (*self.terminal_errors, *self.transient_errors): + expected_exception = self.make_error_exception(error) + with self.subTest(error): + # mock responses + mock_pages = PaginationMockResponse(2*page_size, page_size, pager_type) + mock_first_page = mock_pages.get_next_page() + mock_second_page = mock_pages.get_next_page() + mock_third_page = mock_pages.get_next_page() # empty + for responses in ( + # first sub-test, error on first page + (expected_exception, mock_first_page, mock_second_page, mock_third_page), + # second sub-test error on second page + (mock_first_page, expected_exception, mock_second_page, mock_third_page), + ): + with patch(PaginationMockSupport.operation_map[pager_type], side_effect=iter(responses)): + pager: Pager = Pagination.new_pagination(self.client, pager_type, limit=page_size).pager() + expect_exception_on_page: int = responses.index(expected_exception) + 1 + actual_page_count: int = 0 + while (pager.has_next()): + actual_page_count += 1 + if actual_page_count == expect_exception_on_page: + with self.assertRaises(type(expected_exception), msg='There should be an exception while paging.'): + pager.get_next() + else: + pager.get_next() + + def test_pages_errors(self): + page_size = _MAX_LIMIT + for pager_type in PagerType: + with self.subTest(pager_type): + for error in (*self.terminal_errors, *self.transient_errors): + expected_exception = self.make_error_exception(error) + with self.subTest(error): + for responses in ( + # first sub-test, error on first page + (expected_exception,), + # second sub-test error on second page + (PaginationMockResponse(2*page_size, page_size, pager_type).get_next_page(), expected_exception), + ): + with patch(PaginationMockSupport.operation_map[pager_type], side_effect=iter(responses)): + actual_page_count: int = 0 + expected_page_count: int = len(responses) - 1 + with self.assertRaises(type(expected_exception), msg='There should be an exception while paging.'): + for page in Pagination.new_pagination(self.client, pager_type, limit=page_size).pages(): + actual_page_count += 1 + self.assertEqual(actual_page_count, expected_page_count, 'Should have got the correct number of pages before error.') + + def test_rows_errors(self): + page_size = _MAX_LIMIT + for pager_type in PagerType: + with self.subTest(pager_type): + for error in (*self.terminal_errors, *self.transient_errors): + expected_exception = self.make_error_exception(error) + with self.subTest(error): + for responses in ( + # first sub-test, error on first page + (expected_exception,), + # second sub-test error on second page + (PaginationMockResponse(2*page_size, page_size, pager_type).get_next_page(), expected_exception), + ): + with patch(PaginationMockSupport.operation_map[pager_type], side_effect=iter(responses)): + actual_item_count: int = 0 + expected_item_count: int = page_size * (len(responses) - 1) + with self.assertRaises(type(expected_exception), msg='There should be an exception while paging.'): + for row in Pagination.new_pagination(self.client, pager_type, limit=page_size).rows(): + actual_item_count += 1 + self.assertEqual(actual_item_count, expected_item_count, 'Should have got the correct number of items before error.') From 7ce1c74424f9fc03e732f09dc14a1e728bef37ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Korn=C3=A9l=20T=C3=B3th?= Date: Mon, 30 Jun 2025 15:17:24 +0200 Subject: [PATCH 25/27] docs: pagination examples docs: pagination examples --- .../pagination/all_docs_pagination.py | 60 ++++++++++++++++++ .../pagination/design_docs_pagination.py | 59 ++++++++++++++++++ .../features/pagination/find_pagination.py | 62 +++++++++++++++++++ .../partition_all_docs_pagination.py | 60 ++++++++++++++++++ .../pagination/partition_find_pagination.py | 62 +++++++++++++++++++ .../pagination/partition_search_pagination.py | 58 +++++++++++++++++ .../pagination/partition_view_pagination.py | 62 +++++++++++++++++++ .../features/pagination/search_pagination.py | 57 +++++++++++++++++ .../features/pagination/view_pagination.py | 56 +++++++++++++++++ 9 files changed, 536 insertions(+) create mode 100644 test/examples/src/features/pagination/all_docs_pagination.py create mode 100644 test/examples/src/features/pagination/design_docs_pagination.py create mode 100644 test/examples/src/features/pagination/find_pagination.py create mode 100644 test/examples/src/features/pagination/partition_all_docs_pagination.py create mode 100644 test/examples/src/features/pagination/partition_find_pagination.py create mode 100644 test/examples/src/features/pagination/partition_search_pagination.py create mode 100644 test/examples/src/features/pagination/partition_view_pagination.py create mode 100644 test/examples/src/features/pagination/search_pagination.py create mode 100644 test/examples/src/features/pagination/view_pagination.py diff --git a/test/examples/src/features/pagination/all_docs_pagination.py b/test/examples/src/features/pagination/all_docs_pagination.py new file mode 100644 index 00000000..d1765b98 --- /dev/null +++ b/test/examples/src/features/pagination/all_docs_pagination.py @@ -0,0 +1,60 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'orders', # example database name + 'limit': 50, # limit option sets the page size, + 'start_key': 'abc' # start from example doc ID abc +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_ALL_DOCS, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# Option: use pager next page +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +# Option: use pager all results +# For retrieving all result rows in a single list +# Note: all result rows may be very large! +# Preferably use iterables instead of get_all for memory efficiency with large result sets. +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/design_docs_pagination.py b/test/examples/src/features/pagination/design_docs_pagination.py new file mode 100644 index 00000000..244f03ac --- /dev/null +++ b/test/examples/src/features/pagination/design_docs_pagination.py @@ -0,0 +1,59 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'shoppers', # example database name + 'limit': 50 # limit option sets the page size +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_DESIGN_DOCS, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# Option: use pager next page +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +# Option: use pager all results +# For retrieving all result rows in a single list +# Note: all result rows may be very large! +# Preferably use iterables instead of get_all for memory efficiency with large result sets. +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/find_pagination.py b/test/examples/src/features/pagination/find_pagination.py new file mode 100644 index 00000000..a7b1dae4 --- /dev/null +++ b/test/examples/src/features/pagination/find_pagination.py @@ -0,0 +1,62 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'shoppers', # example database name + 'limit': 50, # limit option sets the page size + 'fields': ['_id', 'type', 'name', 'email'], # return these fields + 'selector': {'email_verified': True}, # select docs with verified emails + 'sort': [{'email': 'desc'}] # sort descending by email +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_FIND, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# Option: use pager next page +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +# Option: use pager all results +# For retrieving all result rows in a single list +# Note: all result rows may be very large! +# Preferably use iterables instead of get_all for memory efficiency with large result sets. +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/partition_all_docs_pagination.py b/test/examples/src/features/pagination/partition_all_docs_pagination.py new file mode 100644 index 00000000..662b95c5 --- /dev/null +++ b/test/examples/src/features/pagination/partition_all_docs_pagination.py @@ -0,0 +1,60 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'events', # example database name + 'limit': 50, # limit option sets the page size + 'partition_key': 'ns1HJS13AMkK', # query only this partition +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_PARTITION_ALL_DOCS, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# Option: use pager next page +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +# Option: use pager all results +# For retrieving all result rows in a single list +# Note: all result rows may be very large! +# Preferably use iterables instead of get_all for memory efficiency with large result sets. +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/partition_find_pagination.py b/test/examples/src/features/pagination/partition_find_pagination.py new file mode 100644 index 00000000..035d680b --- /dev/null +++ b/test/examples/src/features/pagination/partition_find_pagination.py @@ -0,0 +1,62 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'events', # example database name + 'limit': 50, # limit option sets the page size + 'partition_key': 'ns1HJS13AMkK', # query only this partition + 'fields': ['productId', 'eventType', 'date'], # return these fields + 'selector': {'userId': 'abc123'} # select documents with "userId" field equal to "abc123" +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_PARTITION_FIND, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# Option: use pager next page +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +# Option: use pager all results +# For retrieving all result rows in a single list +# Note: all result rows may be very large! +# Preferably use iterables instead of get_all for memory efficiency with large result sets. +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/partition_search_pagination.py b/test/examples/src/features/pagination/partition_search_pagination.py new file mode 100644 index 00000000..134ab4dd --- /dev/null +++ b/test/examples/src/features/pagination/partition_search_pagination.py @@ -0,0 +1,58 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'events', # example database name + 'limit': 50, # limit option sets the page size + 'partition_key': 'ns1HJS13AMkK', # query only this partition + 'ddoc': 'checkout', # use the allUsers design document + 'index': 'findByDate', # search in this index + 'query': 'date:[2019-01-01T12:00:00.000Z TO 2019-01-31T12:00:00.000Z]' # Lucene search query +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_PARTITION_SEARCH, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/partition_view_pagination.py b/test/examples/src/features/pagination/partition_view_pagination.py new file mode 100644 index 00000000..26e70dd1 --- /dev/null +++ b/test/examples/src/features/pagination/partition_view_pagination.py @@ -0,0 +1,62 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'events', # example database name + 'limit': 50, # limit option sets the page size + 'partition_key': 'ns1HJS13AMkK', # query only this partition + 'ddoc': 'checkout', # use the checkout design document + 'view': 'byProductId' # the view to use +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_PARTITION_VIEW, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# Option: use pager next page +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +# Option: use pager all results +# For retrieving all result rows in a single list +# Note: all result rows may be very large! +# Preferably use iterables instead of get_all for memory efficiency with large result sets. +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/search_pagination.py b/test/examples/src/features/pagination/search_pagination.py new file mode 100644 index 00000000..efc3b0e0 --- /dev/null +++ b/test/examples/src/features/pagination/search_pagination.py @@ -0,0 +1,57 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'shoppers', # example database name + 'limit': 50, # limit option sets the page size + 'ddoc': 'allUsers', # use the allUsers design document + 'index': 'activeUsers', # search in this index + 'query': 'name:Jane* AND active:True' # Lucene search query +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_SEARCH, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass diff --git a/test/examples/src/features/pagination/view_pagination.py b/test/examples/src/features/pagination/view_pagination.py new file mode 100644 index 00000000..a0d21495 --- /dev/null +++ b/test/examples/src/features/pagination/view_pagination.py @@ -0,0 +1,56 @@ +# © Copyright IBM Corporation 2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ibmcloudant import Pager, Pagination, PagerType +from ibmcloudant.cloudant_v1 import CloudantV1 + +# Initialize service +service = CloudantV1.new_instance() + +# Setup options +opts = { + 'db': 'shoppers', # example database name + 'limit': 50, # limit option sets the page size + 'ddoc': 'allUsers', # use the allUsers design document + 'view': 'getVerifiedEmails' # the view to use +} + +# Create pagination +pagination = Pagination.new_pagination( + service, PagerType.POST_VIEW, **opts) +# pagination can be reused without side-effects as a factory for iterables or pagers +# options are fixed at pagination creation time + +# Option: iterate pages +# Ideal for using a for loop with each page. +# Each call to pages() returns a fresh iterator that can be traversed once. +for page in pagination.pages(): + # Do something with page + pass + +# Option: iterate rows +# Ideal for using a for loop with each row. +# Each call to rows() returns a fresh iterator that can be traversed once. +for row in pagination.rows(): + # Do something with row + pass + +# For retrieving one page at a time with a method call. +pager: Pager = pagination.pager() +if pager.get_next(): + page = pager.get_next() + # Do something with page + +all_pager: Pager = pagination.pager() +all_rows = all_pager.get_all() +for page in all_rows: + # Do something with row + pass From aed55cae1aa83dd424aaf4cefbad55ce9a53de2b Mon Sep 17 00:00:00 2001 From: Kornel-Toth Date: Tue, 1 Jul 2025 09:55:54 +0200 Subject: [PATCH 26/27] fix: Optional[str] for python 3.9 --- ibmcloudant/features/pagination.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ibmcloudant/features/pagination.py b/ibmcloudant/features/pagination.py index db62f3ac..ce97cde9 100644 --- a/ibmcloudant/features/pagination.py +++ b/ibmcloudant/features/pagination.py @@ -26,7 +26,7 @@ from enum import auto, Enum from functools import partial from types import MappingProxyType -from typing import Generic, Protocol, TypeVar +from typing import Generic, Optional, Protocol, TypeVar from ibm_cloud_sdk_core import DetailedResponse from ibmcloudant.cloudant_v1 import CloudantV1,\ @@ -307,7 +307,7 @@ class _KeyPageIterator(_BasePageIterator, Generic[K]): def __init__(self, client: CloudantV1, operation: Callable[..., DetailedResponse], opts: dict): super().__init__(client, operation, ['start_key', 'start_key_doc_id'], opts) - self._boundary_failure: str | None = None + self._boundary_failure: Optional[str] = None def _next_request(self) -> list[I]: if self._boundary_failure is not None: @@ -318,7 +318,7 @@ def _next_request(self) -> list[I]: if len(items) > 0: # Get, but don't remove the last item from the list penultimate_item: I = items[-1] - self._boundary_failure: str | None = self.check_boundary(penultimate_item, last_item) + self._boundary_failure: Optional[str] = self.check_boundary(penultimate_item, last_item) return items def _get_next_page_options(self, result: R) -> dict: @@ -336,7 +336,7 @@ def _page_size_from_opts_limit(self, opts:dict) -> int: return super()._page_size_from_opts_limit(opts) + 1 @abstractmethod - def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: + def check_boundary(self, penultimate_item: I, last_item: I) -> Optional[str]: raise NotImplementedError() class _BookmarkPageIterator(_BasePageIterator): @@ -358,7 +358,7 @@ def _get_next_page_options(self, result: R) -> dict: del opts['start_key_doc_id'] return opts - def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: + def check_boundary(self, penultimate_item: I, last_item: I) -> Optional[str]: # IDs are always unique in _all_docs pagers so return None return None @@ -418,7 +418,7 @@ class _ViewBasePageIterator(_KeyPageIterator[any]): def _result_converter(self): return ViewResult.from_dict - def check_boundary(self, penultimate_item: I, last_item: I) -> str | None: + def check_boundary(self, penultimate_item: I, last_item: I) -> Optional[str]: if penultimate_item.id == (boundary_id := last_item.id) \ and penultimate_item.key == (boundary_key := last_item.key): return f'Cannot paginate on a boundary containing identical keys {boundary_key} and document IDs {boundary_id}' From ce40d491e5ada018d4840a908da1e6c2e3a3ae59 Mon Sep 17 00:00:00 2001 From: Kornel-Toth Date: Tue, 1 Jul 2025 12:18:23 +0200 Subject: [PATCH 27/27] test: itertools.batched fix for older versions --- test/unit/features/conftest.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/unit/features/conftest.py b/test/unit/features/conftest.py index a76351fc..dab13b99 100644 --- a/test/unit/features/conftest.py +++ b/test/unit/features/conftest.py @@ -28,6 +28,7 @@ from threading import Timer import itertools +from itertools import islice from requests import codes from requests.exceptions import ConnectionError @@ -337,8 +338,18 @@ def __init__(self, self.plus_one_paging: bool = self.pager_type in PaginationMockSupport.key_pagers self.expected_pages: list[list] = [] + # for compatibility with python <= 3.12 + def batched(self, iterable, n): + """Batch data into tuples of length n. The last batch may be shorter.""" + it = iter(iterable) + while True: + batch = tuple(islice(it, n)) + if not batch: + break + yield batch + def generator(self): - for page in itertools.batched(range(0, self.total_items), self.page_size): + for page in self.batched(range(0, self.total_items), self.page_size): rows = [PaginationMockSupport.make_row(self.pager_type, i) for i in page] if self.plus_one_paging: # Add an n+1 row for key based paging if more pages