service: Support for partial listings (server-side pagination)

Reading the spec, using the __next field, not $top/$skip/__count seems
to be the pristine way of fetching an entity collection in full:

> In response payloads, representing Collections of Entries, if the
> server does not include an object for every Entry in the Collection of
> Entries identified by the request URI then the response represents a
> partial listings of the Collection. In this case, "__next" name/value
> pair is included to indicate the response represents a partial
> listing. The value of the name/value pair is a URI which identifies
> the next partial set of entities from the originally identified
> complete set.

Author: rettichschnidi

pyodata/v2/service.py

@@ -238,6 +238,7 @@ def __init__(self, url, connection, handler, headers=None):
         self._headers = headers or dict()
         self._logger = logging.getLogger(LOGGER_NAME)
         self._customs = {}  # string -> string hash
+        self._next_url = None
 
     @property
     def handler(self):
@@ -299,7 +300,10 @@ def execute(self):
 
            Fetches HTTP response and returns processed result"""
 
-        url = urljoin(self._url, self.get_path())
+        if self._next_url:
+            url = self._next_url
+        else:
+            url = urljoin(self._url, self.get_path())
         # pylint: disable=assignment-from-none
         body = self.get_body()
 
@@ -616,6 +620,17 @@ def count(self, inline=False):
             self._count = True
         return self
 
+    def next_url(self, next_url):
+        """
+        Sets URL which identifies the next partial set of entities from the originally identified complete set. Once
+        set, this URL takes precedence over all query parameters.
+
+        For details, see section "6. Representing Collections of Entries" on
+        https://www.odata.org/documentation/odata-version-2-0/json-format/
+        """
+        self._next_url = next_url
+        return self
+
     def expand(self, expand):
         """Sets the expand expressions."""
         self._expand = expand
@@ -667,6 +682,9 @@ def get_default_headers(self):
         }
 
     def get_query_params(self):
+        if self._next_url:
+            return {}
+
         qparams = super(QueryRequest, self).get_query_params()
 
         if self._top is not None:
@@ -1250,11 +1268,24 @@ def filter(self, *args, **kwargs):
 
 
 class ListWithTotalCount(list):
-    """A list with the additional property total_count"""
+    """
+    A list with the additional property total_count and next_url.
+
+    If set, use next_url to fetch the next batch of entities.
+    """
 
-    def __init__(self, total_count):
+    def __init__(self, total_count, next_url):
         super(ListWithTotalCount, self).__init__()
         self._total_count = total_count
+        self._next_url = next_url
+
+    @property
+    def next_url(self):
+        """
+        URL which identifies the next partial set of entities from the originally identified complete set. None if no
+        entities remaining.
+        """
+        return self._next_url
 
     @property
     def total_count(self):
@@ -1390,7 +1421,8 @@ def get_entity_handler(response):
         return EntityGetRequest(get_entity_handler, entity_key, self)
 
     def get_entities(self):
-        """Get all entities"""
+        """Get some, potentially all entities"""
+
         def get_entities_handler(response):
             """Gets entity set from HTTP Response"""
 
@@ -1405,15 +1437,18 @@ def get_entities_handler(response):
 
             entities = content['d']
             total_count = None
+            next_url = None
 
             if isinstance(entities, dict):
                 if '__count' in entities:
                     total_count = int(entities['__count'])
+                if '__next' in entities:
+                    next_url = entities['__next']
                 entities = entities['results']
 
             self._logger.info('Fetched %d entities', len(entities))
 
-            result = ListWithTotalCount(total_count)
+            result = ListWithTotalCount(total_count, next_url)
             for props in entities:
                 entity = EntityProxy(self._service, self._entity_set, self._entity_set.entity_type, props)
                 result.append(entity)

tests/test_service_v2.py

@@ -2005,6 +2005,64 @@ def test_count_with_chainable_filter(service):
     assert request.execute() == 3
 
 
+@responses.activate
+def test_partial_listing(service):
+    """Using __next URI to fetch all entities in a collection"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        f"{service.url}/Employees?$inlinecount=allpages",
+        json={'d': {
+            '__count': 3,
+            '__next': f"{service.url}/Employees?$inlinecount=allpages&$skiptoken='opaque'",
+            'results': [
+                {
+                    'ID': 21,
+                    'NameFirst': 'George',
+                    'NameLast': 'Doe'
+                },{
+                    'ID': 22,
+                    'NameFirst': 'John',
+                    'NameLast': 'Doe'
+                }
+            ]
+        }},
+        status=200)
+
+    responses.add(
+        responses.GET,
+        f"{service.url}/Employees?$inlinecount=allpages&$skiptoken='opaque'",
+        json={'d': {
+            '__count': 3,
+            'results': [
+                {
+                    'ID': 23,
+                    'NameFirst': 'Rob',
+                    'NameLast': 'Ickes'
+                }
+            ]
+        }},
+        status=200)
+
+    # Fetching (potentially) all entities, actually getting 2
+    request = service.entity_sets.Employees.get_entities().count(inline=True)
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+    result = request.execute()
+    assert len(result) == 2
+    assert result.total_count == 3
+    assert result.next_url is not None
+
+    # Fetching next batch, receive the one remaining entity
+    request = service.entity_sets.Employees.get_entities().next_url(result.next_url)
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+    result = request.execute()
+    assert len(result) == 1
+    assert result.total_count == 3, "(inline) count flag inherited from first request"
+    assert result.next_url is None
+
+
 @responses.activate
 def test_count_with_chainable_filter_lt_operator(service):
     """Check getting $count with $filter with new filter syntax using multiple filters"""