From 152a2273cb32400a1203d6c9e8961258211ab450 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 2 Jan 2025 14:56:02 -0500 Subject: [PATCH 01/35] feat: adding input parameters to endpoint functions for /orders for pagination tests: creating incomplete test to start testing pagination --- src/stapi_fastapi/backends/root_backend.py | 4 +- src/stapi_fastapi/models/order.py | 5 ++ src/stapi_fastapi/routers/root_router.py | 6 ++- tests/application.py | 4 +- tests/test_order.py | 56 ++++++++++++++++++++++ 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 9a803d9..aba5d3d 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -13,7 +13,9 @@ class RootBackend[T: OrderStatusPayload, U: OrderStatus](Protocol): # pragma: nocover - async def get_orders(self, request: Request) -> ResultE[OrderCollection]: + async def get_orders( + self, request: Request, next_token: str, limit: int + ) -> ResultE[OrderCollection]: """ Return a list of existing orders. """ diff --git a/src/stapi_fastapi/models/order.py b/src/stapi_fastapi/models/order.py index ba379ca..239f8fd 100644 --- a/src/stapi_fastapi/models/order.py +++ b/src/stapi_fastapi/models/order.py @@ -113,6 +113,11 @@ def __getitem__(self, index: int) -> Order: return self.features[index] +class Orders(BaseModel): + collection: OrderCollection + token: str + + class OrderPayload(BaseModel, Generic[ORP]): datetime: DatetimeInterval geometry: Geometry diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index f3e5faa..b5ff4a0 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -159,8 +159,10 @@ def get_products(self, request: Request) -> ProductsCollection: ], ) - async def get_orders(self, request: Request) -> OrderCollection: - match await self.backend.get_orders(request): + async def get_orders( + self, request: Request, next_token: str, limit: int + ) -> OrderCollection: + match await self.backend.get_orders(request, next_token, limit): case Success(orders): for order in orders: order.links.append( diff --git a/tests/application.py b/tests/application.py index 6e9afaf..e23ddba 100644 --- a/tests/application.py +++ b/tests/application.py @@ -43,7 +43,9 @@ class MockRootBackend(RootBackend): def __init__(self, orders: InMemoryOrderDB) -> None: self._orders_db: InMemoryOrderDB = orders - async def get_orders(self, request: Request) -> ResultE[OrderCollection]: + async def get_orders( + self, request: Request, next_token: str, limit: int + ) -> ResultE[OrderCollection]: """ Show all orders. """ diff --git a/tests/test_order.py b/tests/test_order.py index d795634..0f47c59 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -168,3 +168,59 @@ def test_order_status_after_update( assert s["reason_text"] is None assert s["status_code"] == "received" assert s["timestamp"] + + +@pytest.fixture +def create_second_order_allowed_payloads() -> list[OrderPayload]: + return [ + OrderPayload( + geometry=Point( + type="Point", coordinates=Position2D(longitude=14.4, latitude=56.5) + ), + datetime=( + datetime.fromisoformat("2024-10-09T18:55:33Z"), + datetime.fromisoformat("2024-10-12T18:55:33Z"), + ), + filter=None, + order_parameters=MyOrderParameters(s3_path="s3://my-bucket"), + ), + ] + + +@pytest.mark.parametrize("product_id", ["test-spotlight"]) +def test_order_pagination( + product_id: str, + product_backend: MockProductBackend, + stapi_client: TestClient, + create_order_allowed_payloads: list[OrderPayload], + create_second_order_allowed_payloads: list[OrderPayload], +) -> None: + product_backend._allowed_payloads = create_order_allowed_payloads + + # check empty + res = stapi_client.get("/orders", params={"next_token": "a", "limit": 10}) + default_orders = {"type": "FeatureCollection", "features": [], "links": []} + + assert res.status_code == status.HTTP_200_OK + assert res.headers["Content-Type"] == "application/geo+json" + assert res.json() == default_orders + + # add order to product + res = stapi_client.post( + f"products/{product_id}/orders", + json=create_order_allowed_payloads[0].model_dump(), + ) + + assert res.status_code == status.HTTP_201_CREATED, res.text + assert res.headers["Content-Type"] == "application/geo+json" + + res = stapi_client.post( + f"products/{product_id}/orders", + json=create_second_order_allowed_payloads[0].model_dump(), + ) + + # call all orders + res = stapi_client.get("/orders", params={"next_token": "a", "limit": 1}) + assert res.status_code == status.HTTP_200_OK + assert res.headers["Content-Type"] == "application/geo+json" + print("hold") From 0dcf05becc6d985114e3396771371b9078be022e Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 2 Jan 2025 15:28:36 -0500 Subject: [PATCH 02/35] feat: applying new Orders class and test is breaking in the expected, desired way --- src/stapi_fastapi/backends/root_backend.py | 4 ++-- src/stapi_fastapi/routers/root_router.py | 4 ++-- tests/application.py | 12 ++++++++++-- tests/test_order.py | 8 +++++--- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index aba5d3d..1ba7f01 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -6,7 +6,7 @@ from stapi_fastapi.models.order import ( Order, - OrderCollection, + Orders, OrderStatus, OrderStatusPayload, ) @@ -15,7 +15,7 @@ class RootBackend[T: OrderStatusPayload, U: OrderStatus](Protocol): # pragma: nocover async def get_orders( self, request: Request, next_token: str, limit: int - ) -> ResultE[OrderCollection]: + ) -> ResultE[Orders]: """ Return a list of existing orders. """ diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index b5ff4a0..8d1c0d3 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -164,7 +164,7 @@ async def get_orders( ) -> OrderCollection: match await self.backend.get_orders(request, next_token, limit): case Success(orders): - for order in orders: + for order in orders.collection: order.links.append( Link( href=str( @@ -176,7 +176,7 @@ async def get_orders( type=TYPE_JSON, ) ) - return orders + return orders.collection case Failure(e): logging.exception("An error occurred while retrieving orders", e) raise HTTPException( diff --git a/tests/application.py b/tests/application.py index e23ddba..f8904b5 100644 --- a/tests/application.py +++ b/tests/application.py @@ -21,6 +21,7 @@ OrderCollection, OrderParameters, OrderPayload, + Orders, OrderStatus, OrderStatusCode, OrderStatusPayload, @@ -45,11 +46,18 @@ def __init__(self, orders: InMemoryOrderDB) -> None: async def get_orders( self, request: Request, next_token: str, limit: int - ) -> ResultE[OrderCollection]: + ) -> ResultE[Orders]: """ Show all orders. """ - return Success(OrderCollection(features=list(self._orders_db._orders.values()))) + return Success( + Orders( + collection=OrderCollection( + features=list(self._orders_db._orders.values()) + ), + token="a", + ) + ) async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Order]]: """ diff --git a/tests/test_order.py b/tests/test_order.py index 0f47c59..841da5e 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -198,7 +198,7 @@ def test_order_pagination( product_backend._allowed_payloads = create_order_allowed_payloads # check empty - res = stapi_client.get("/orders", params={"next_token": "a", "limit": 10}) + res = stapi_client.get("/orders", params={"next_token": "a", "limit": 1}) default_orders = {"type": "FeatureCollection", "features": [], "links": []} assert res.status_code == status.HTTP_200_OK @@ -221,6 +221,8 @@ def test_order_pagination( # call all orders res = stapi_client.get("/orders", params={"next_token": "a", "limit": 1}) + checker = res.json() + assert res.status_code == status.HTTP_200_OK - assert res.headers["Content-Type"] == "application/geo+json" - print("hold") + assert len(checker["features"]) == 1 + assert checker["links"] != [] From 7734317d45ada17c7bdea86aaa1d8f8713babc12 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 3 Jan 2025 11:13:21 -0500 Subject: [PATCH 03/35] feat: correctly adding pagination link object to top level collection feat: replacing token in url if new token is provided tests: continue to updae test to validate additional pieces of the limit/token expected functionality, still breaking on limit check feat: removing use or Orders type and going with tuple instead --- src/stapi_fastapi/backends/root_backend.py | 9 +++++---- src/stapi_fastapi/models/order.py | 5 ----- src/stapi_fastapi/routers/root_router.py | 23 ++++++++++++++++++---- tests/test_order.py | 16 ++++++++++++--- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 1ba7f01..37d8880 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -6,7 +6,7 @@ from stapi_fastapi.models.order import ( Order, - Orders, + OrderCollection, OrderStatus, OrderStatusPayload, ) @@ -14,10 +14,11 @@ class RootBackend[T: OrderStatusPayload, U: OrderStatus](Protocol): # pragma: nocover async def get_orders( - self, request: Request, next_token: str, limit: int - ) -> ResultE[Orders]: + self, request: Request, next: str | None, limit: int | None + ) -> ResultE[tuple[OrderCollection, str]]: """ - Return a list of existing orders. + Return a list of existing orders and pagination token if applicable + No pagination will return empty string for token """ ... diff --git a/src/stapi_fastapi/models/order.py b/src/stapi_fastapi/models/order.py index 239f8fd..ba379ca 100644 --- a/src/stapi_fastapi/models/order.py +++ b/src/stapi_fastapi/models/order.py @@ -113,11 +113,6 @@ def __getitem__(self, index: int) -> Order: return self.features[index] -class Orders(BaseModel): - collection: OrderCollection - token: str - - class OrderPayload(BaseModel, Generic[ORP]): datetime: DatetimeInterval geometry: Geometry diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 8d1c0d3..d874bb3 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -160,11 +160,11 @@ def get_products(self, request: Request) -> ProductsCollection: ) async def get_orders( - self, request: Request, next_token: str, limit: int + self, request: Request, next_token: str | None = None, limit: int | None = None ) -> OrderCollection: match await self.backend.get_orders(request, next_token, limit): - case Success(orders): - for order in orders.collection: + case Success((collections, token)): + for order in collections: order.links.append( Link( href=str( @@ -176,7 +176,22 @@ async def get_orders( type=TYPE_JSON, ) ) - return orders.collection + if next_token: + query = request.url.components.query + if query: # check url for params + params = { + param.split("=")[0]: param.split("=")[1] + for param in query.split("&") + } + params["next_token"] = token # replace old token if exists + updated_url = request.url.replace_query_params(**params) + else: # add if doesn't exist + updated_url = request.url.include_query_params(token=token) + + collections.links.append( + Link(href=str(updated_url), rel="next", type=TYPE_JSON) + ) + return collections case Failure(e): logging.exception("An error occurred while retrieving orders", e) raise HTTPException( diff --git a/tests/test_order.py b/tests/test_order.py index 841da5e..db30d6d 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -196,9 +196,11 @@ def test_order_pagination( create_second_order_allowed_payloads: list[OrderPayload], ) -> None: product_backend._allowed_payloads = create_order_allowed_payloads + OLD_TOKEN = "a_token" # check empty - res = stapi_client.get("/orders", params={"next_token": "a", "limit": 1}) + res = stapi_client.get("/orders") + default_orders = {"type": "FeatureCollection", "features": [], "links": []} assert res.status_code == status.HTTP_200_OK @@ -220,9 +222,17 @@ def test_order_pagination( ) # call all orders - res = stapi_client.get("/orders", params={"next_token": "a", "limit": 1}) + res = stapi_client.get("/orders", params={"next_token": OLD_TOKEN, "limit": 1}) checker = res.json() assert res.status_code == status.HTTP_200_OK - assert len(checker["features"]) == 1 + + # temp check to make sure token link isn't added to inside collection + for link in checker["features"][1]["links"]: + assert link["rel"] != "next" assert checker["links"] != [] + + # check to make sure new token in link + assert OLD_TOKEN not in checker["links"][0]["href"] + + assert len(checker["features"]) == 1 From 16f4dc21447d1ecdf5825bae823a8bb272c0c923 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 3 Jan 2025 15:24:42 -0500 Subject: [PATCH 04/35] feat: tighened up logic in root_router.py to only add pagination link object if we are returned a token. tests: created simple test implementation to abstract out token handling and generation --- src/stapi_fastapi/routers/root_router.py | 22 +++++------ tests/application.py | 50 +++++++++++++++++++----- tests/test_order.py | 4 +- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index d874bb3..9c481a2 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -160,9 +160,9 @@ def get_products(self, request: Request) -> ProductsCollection: ) async def get_orders( - self, request: Request, next_token: str | None = None, limit: int | None = None + self, request: Request, next: str | None = None, limit: int | None = None ) -> OrderCollection: - match await self.backend.get_orders(request, next_token, limit): + match await self.backend.get_orders(request, next, limit): case Success((collections, token)): for order in collections: order.links.append( @@ -176,18 +176,14 @@ async def get_orders( type=TYPE_JSON, ) ) - if next_token: + if token: # only return pagination link if backend returns token query = request.url.components.query - if query: # check url for params - params = { - param.split("=")[0]: param.split("=")[1] - for param in query.split("&") - } - params["next_token"] = token # replace old token if exists - updated_url = request.url.replace_query_params(**params) - else: # add if doesn't exist - updated_url = request.url.include_query_params(token=token) - + params = { + param.split("=")[0]: param.split("=")[1] + for param in query.split("&") + } + params["next"] = token # replace/add token + updated_url = request.url.replace_query_params(**params) collections.links.append( Link(href=str(updated_url), rel="next", type=TYPE_JSON) ) diff --git a/tests/application.py b/tests/application.py index f8904b5..06f8cb4 100644 --- a/tests/application.py +++ b/tests/application.py @@ -21,7 +21,6 @@ OrderCollection, OrderParameters, OrderPayload, - Orders, OrderStatus, OrderStatusCode, OrderStatusPayload, @@ -45,18 +44,51 @@ def __init__(self, orders: InMemoryOrderDB) -> None: self._orders_db: InMemoryOrderDB = orders async def get_orders( - self, request: Request, next_token: str, limit: int - ) -> ResultE[Orders]: + self, request: Request, next: str | None = None, limit: int | None = None + ) -> ResultE[tuple[OrderCollection, str]]: """ Show all orders. """ - return Success( - Orders( - collection=OrderCollection( - features=list(self._orders_db._orders.values()) - ), - token="a", + # it limit does NOT reach the last index in the db list THEN we return token + + # if no limit - no token since getting all records - return no token + # backend determines if we return a token + if next: # initial implementation - if given a token, assume we continue pagination and return a new token + features = list(self._orders_db._orders.values()) + + # get records based on limit + def token_processor(next: str) -> tuple[str, int]: + """process token to return new token and start loc""" + return "new_token", 0 + + token, start = token_processor(next) + if limit: + if start + limit > len(features): + features = features[start:] + else: + features = features[start:limit] + return Success((OrderCollection(features=features), token)) + else: # if no limit with token + return Success((OrderCollection(features=features[start:]), "")) + else: # no input token means give us everything and return no token + return Success( + (OrderCollection(features=list(self._orders_db._orders.values())), "") ) + # need to be agnostic to token here. If we have MORE records we COULD return - i.e. final index found by limit, then we return a token. If we return the last record THEN return EMPTY token. + # token = '' + # if not limit: + # limit = 0 + # # parse token here and do stuff to get starting index + # start_index = 0 + # end_index = start_index + limit + # if not limit: + # collection = OrderCollection(features=list(self._orders_db._orders.values())) + # else: + # all_orders = list(self._orders_db._orders.values()) + # features = [all_orders[i] for i in indicies if 0 <= i < len(lst)] + # collection = + return Success( + (OrderCollection(features=list(self._orders_db._orders.values())), "") ) async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Order]]: diff --git a/tests/test_order.py b/tests/test_order.py index db30d6d..046f1cf 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -222,13 +222,13 @@ def test_order_pagination( ) # call all orders - res = stapi_client.get("/orders", params={"next_token": OLD_TOKEN, "limit": 1}) + res = stapi_client.get("/orders", params={"next": OLD_TOKEN, "limit": 1}) checker = res.json() assert res.status_code == status.HTTP_200_OK # temp check to make sure token link isn't added to inside collection - for link in checker["features"][1]["links"]: + for link in checker["features"][0]["links"]: assert link["rel"] != "next" assert checker["links"] != [] From e0fa4e7c6eddb1ac6191f0df1a667a099d01d938 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 3 Jan 2025 15:44:46 -0500 Subject: [PATCH 05/35] poetry.lock --- src/stapi_fastapi/routers/root_router.py | 4 +-- tests/application.py | 33 +++++------------------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 9c481a2..d976336 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -176,13 +176,13 @@ async def get_orders( type=TYPE_JSON, ) ) - if token: # only return pagination link if backend returns token + if token: # pagination link if backend returns token query = request.url.components.query params = { param.split("=")[0]: param.split("=")[1] for param in query.split("&") } - params["next"] = token # replace/add token + params["next"] = token updated_url = request.url.replace_query_params(**params) collections.links.append( Link(href=str(updated_url), rel="next", type=TYPE_JSON) diff --git a/tests/application.py b/tests/application.py index 06f8cb4..a8f0b0a 100644 --- a/tests/application.py +++ b/tests/application.py @@ -47,16 +47,13 @@ async def get_orders( self, request: Request, next: str | None = None, limit: int | None = None ) -> ResultE[tuple[OrderCollection, str]]: """ - Show all orders. + Return orders from backend. Handle pagination/limit if applicable """ - # it limit does NOT reach the last index in the db list THEN we return token + features = list(self._orders_db._orders.values()) + if limit and not next: + return Success((OrderCollection(features=features[:limit]), "")) + if next: - # if no limit - no token since getting all records - return no token - # backend determines if we return a token - if next: # initial implementation - if given a token, assume we continue pagination and return a new token - features = list(self._orders_db._orders.values()) - - # get records based on limit def token_processor(next: str) -> tuple[str, int]: """process token to return new token and start loc""" return "new_token", 0 @@ -68,28 +65,12 @@ def token_processor(next: str) -> tuple[str, int]: else: features = features[start:limit] return Success((OrderCollection(features=features), token)) - else: # if no limit with token + else: # token and no limit return Success((OrderCollection(features=features[start:]), "")) - else: # no input token means give us everything and return no token + else: return Success( (OrderCollection(features=list(self._orders_db._orders.values())), "") ) - # need to be agnostic to token here. If we have MORE records we COULD return - i.e. final index found by limit, then we return a token. If we return the last record THEN return EMPTY token. - # token = '' - # if not limit: - # limit = 0 - # # parse token here and do stuff to get starting index - # start_index = 0 - # end_index = start_index + limit - # if not limit: - # collection = OrderCollection(features=list(self._orders_db._orders.values())) - # else: - # all_orders = list(self._orders_db._orders.values()) - # features = [all_orders[i] for i in indicies if 0 <= i < len(lst)] - # collection = - return Success( - (OrderCollection(features=list(self._orders_db._orders.values())), "") - ) async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Order]]: """ From 90d0ebdd0e7f78e7d1d0f6dd0ee054ec3529637e Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 3 Jan 2025 15:48:27 -0500 Subject: [PATCH 06/35] feat: updating lock file header fix: fix previous commit message --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0c692dd..59bfa29 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1183,4 +1183,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" python-versions = "3.12.*" -content-hash = "d3c6efb6d8f4a49b8098ce6aa8a383ede2961b5632e66b6e56f5c86ed2edf8c9" +content-hash = "dbd480b7af18b692724040c7f01a5e302a61564bda31876482f4f0917b3244de" From 53afb6664b1e555325aead81521ffdf977817aaf Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Mon, 6 Jan 2025 22:08:17 -0500 Subject: [PATCH 07/35] feat: rework test backend to make code smoother feat: use default feat: use include_query_params instead of parsing query param string ourselves --- src/stapi_fastapi/backends/root_backend.py | 2 +- src/stapi_fastapi/routers/root_router.py | 10 +---- tests/application.py | 48 +++++++++++----------- tests/test_order.py | 9 ++-- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 37d8880..f0635a2 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -14,7 +14,7 @@ class RootBackend[T: OrderStatusPayload, U: OrderStatus](Protocol): # pragma: nocover async def get_orders( - self, request: Request, next: str | None, limit: int | None + self, request: Request, next: str | None, limit: int ) -> ResultE[tuple[OrderCollection, str]]: """ Return a list of existing orders and pagination token if applicable diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index d976336..f374404 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -160,7 +160,7 @@ def get_products(self, request: Request) -> ProductsCollection: ) async def get_orders( - self, request: Request, next: str | None = None, limit: int | None = None + self, request: Request, next: str | None = None, limit: int = 10 ) -> OrderCollection: match await self.backend.get_orders(request, next, limit): case Success((collections, token)): @@ -177,13 +177,7 @@ async def get_orders( ) ) if token: # pagination link if backend returns token - query = request.url.components.query - params = { - param.split("=")[0]: param.split("=")[1] - for param in query.split("&") - } - params["next"] = token - updated_url = request.url.replace_query_params(**params) + updated_url = request.url.include_query_params(next=token) collections.links.append( Link(href=str(updated_url), rel="next", type=TYPE_JSON) ) diff --git a/tests/application.py b/tests/application.py index a8f0b0a..a177b50 100644 --- a/tests/application.py +++ b/tests/application.py @@ -44,33 +44,35 @@ def __init__(self, orders: InMemoryOrderDB) -> None: self._orders_db: InMemoryOrderDB = orders async def get_orders( - self, request: Request, next: str | None = None, limit: int | None = None + self, request: Request, next: str | None, limit: int ) -> ResultE[tuple[OrderCollection, str]]: """ Return orders from backend. Handle pagination/limit if applicable """ - features = list(self._orders_db._orders.values()) - if limit and not next: - return Success((OrderCollection(features=features[:limit]), "")) - if next: - - def token_processor(next: str) -> tuple[str, int]: - """process token to return new token and start loc""" - return "new_token", 0 - - token, start = token_processor(next) - if limit: - if start + limit > len(features): - features = features[start:] - else: - features = features[start:limit] - return Success((OrderCollection(features=features), token)) - else: # token and no limit - return Success((OrderCollection(features=features[start:]), "")) - else: - return Success( - (OrderCollection(features=list(self._orders_db._orders.values())), "") - ) + try: + start = 0 + order_ids = sorted(self._orders_db._orders.keys()) + if not order_ids: # not data in db return empty + return Success( + ( + OrderCollection( + features=list(self._orders_db._orders.values()) + ), + "", + ) + ) + + if next: + start = [i for i, x in enumerate(order_ids) if x == next][0] + end = start + limit + ids = order_ids[start:end] + + feats = [self._orders_db._orders[order_id] for order_id in ids] + next = self._orders_db._orders[order_ids[end]].id + + return Success((OrderCollection(features=feats), next)) + except Exception as e: + return Failure(e) async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Order]]: """ diff --git a/tests/test_order.py b/tests/test_order.py index 046f1cf..615d445 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -187,7 +187,7 @@ def create_second_order_allowed_payloads() -> list[OrderPayload]: ] -@pytest.mark.parametrize("product_id", ["test-spotlight"]) +@pytest.mark.parametrize("product_id", [pytest.param("test-spotlight", id="base test")]) def test_order_pagination( product_id: str, product_backend: MockProductBackend, @@ -196,7 +196,6 @@ def test_order_pagination( create_second_order_allowed_payloads: list[OrderPayload], ) -> None: product_backend._allowed_payloads = create_order_allowed_payloads - OLD_TOKEN = "a_token" # check empty res = stapi_client.get("/orders") @@ -220,9 +219,9 @@ def test_order_pagination( f"products/{product_id}/orders", json=create_second_order_allowed_payloads[0].model_dump(), ) - # call all orders - res = stapi_client.get("/orders", params={"next": OLD_TOKEN, "limit": 1}) + next = res.json()["id"] + res = stapi_client.get("/orders", params={"next": next, "limit": 1}) checker = res.json() assert res.status_code == status.HTTP_200_OK @@ -233,6 +232,6 @@ def test_order_pagination( assert checker["links"] != [] # check to make sure new token in link - assert OLD_TOKEN not in checker["links"][0]["href"] + assert next not in checker["links"][0]["href"] assert len(checker["features"]) == 1 From 723e645abcb66bb3a1c2363a038c6c2e67a64515 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Wed, 8 Jan 2025 13:16:15 -0500 Subject: [PATCH 08/35] feat: update Failure logic in root_router get_orders to return 404 for bad token tests: reworking pagination tests tests: adding handle to max limit in test implemented backend --- src/stapi_fastapi/routers/root_router.py | 13 +- tests/application.py | 12 +- tests/test_order.py | 153 ++++++++++++++++------- 3 files changed, 123 insertions(+), 55 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index f374404..0e8753a 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -183,11 +183,14 @@ async def get_orders( ) return collections case Failure(e): - logging.exception("An error occurred while retrieving orders", e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error finding Orders", - ) + logging.exception(f"An error occurred while retrieving orders: {e}") + if isinstance(e, IndexError): + raise NotFoundException(detail="Error finding pagination token") + else: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error finding Orders", + ) case _: raise AssertionError("Expected code to be unreachable") diff --git a/tests/application.py b/tests/application.py index a177b50..917260d 100644 --- a/tests/application.py +++ b/tests/application.py @@ -51,8 +51,11 @@ async def get_orders( """ try: start = 0 - order_ids = sorted(self._orders_db._orders.keys()) - if not order_ids: # not data in db return empty + if limit > 100: + limit = 100 + + order_ids = [*self._orders_db._orders.keys()] + if not order_ids: # no data in db return Success( ( OrderCollection( @@ -66,10 +69,11 @@ async def get_orders( start = [i for i, x in enumerate(order_ids) if x == next][0] end = start + limit ids = order_ids[start:end] - feats = [self._orders_db._orders[order_id] for order_id in ids] - next = self._orders_db._orders[order_ids[end]].id + next = "" + if end < len(order_ids): + next = self._orders_db._orders[order_ids[end]].id return Success((OrderCollection(features=feats), next)) except Exception as e: return Failure(e) diff --git a/tests/test_order.py b/tests/test_order.py index 615d445..0de5c5d 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -171,67 +171,128 @@ def test_order_status_after_update( @pytest.fixture -def create_second_order_allowed_payloads() -> list[OrderPayload]: - return [ - OrderPayload( +def create_order_payloads() -> list[OrderPayload]: + datetimes = [ + ("2024-10-09T18:55:33Z", "2024-10-12T18:55:33Z"), + ("2024-10-15T18:55:33Z", "2024-10-18T18:55:33Z"), + ("2024-10-20T18:55:33Z", "2024-10-23T18:55:33Z"), + ] + payloads = [] + for start, end in datetimes: + payload = OrderPayload( geometry=Point( type="Point", coordinates=Position2D(longitude=14.4, latitude=56.5) ), datetime=( - datetime.fromisoformat("2024-10-09T18:55:33Z"), - datetime.fromisoformat("2024-10-12T18:55:33Z"), + datetime.fromisoformat(start), + datetime.fromisoformat(end), ), filter=None, order_parameters=MyOrderParameters(s3_path="s3://my-bucket"), - ), - ] + ) + payloads.append(payload) + return payloads -@pytest.mark.parametrize("product_id", [pytest.param("test-spotlight", id="base test")]) -def test_order_pagination( - product_id: str, - product_backend: MockProductBackend, +@pytest.fixture +def prepare_order_pagination( + stapi_client: TestClient, create_order_payloads: list[OrderPayload] +) -> tuple[str, str, str]: + # product_backend._allowed_payloads = create_order_payloads + product_id = "test-spotlight" + + # # check empty + # res = stapi_client.get("/orders") + # default_orders = {"type": "FeatureCollection", "features": [], "links": []} + # assert res.status_code == status.HTTP_200_OK + # assert res.headers["Content-Type"] == "application/geo+json" + # assert res.json() == default_orders + + # get uuids created to use as pagination tokens + order_ids = [] + for payload in create_order_payloads: + res = stapi_client.post( + f"products/{product_id}/orders", + json=payload.model_dump(), + ) + assert res.status_code == status.HTTP_201_CREATED, res.text + assert res.headers["Content-Type"] == "application/geo+json" + order_ids.append(res.json()["id"]) + + # res = stapi_client.get("/orders") + # checker = res.json() + # assert len(checker['features']) == 3 + + return tuple(order_ids) + + +@pytest.mark.parametrize( + "product_id,expected_status,limit,id_retrieval,token_back", + [ + pytest.param( + "test-spotlight", + status.HTTP_200_OK, + 1, + 0, + True, + id="input frst order_id token get new token back", + ), + pytest.param( + "test-spotlight", + status.HTTP_200_OK, + 1, + 2, + False, + id="input last order_id token get NO token back", + ), + pytest.param( + "test-spotlight", + status.HTTP_404_NOT_FOUND, + 1, + "BAD_TOKEN", + False, + id="input bad token get 404 back", + ), + pytest.param( + "test-spotlight", + status.HTTP_200_OK, + 1, + 1000000, + False, + id="high limit handled and returns valid records", + ), + ], +) +def test_order_pagination_hold( + prepare_order_pagination, stapi_client: TestClient, - create_order_allowed_payloads: list[OrderPayload], - create_second_order_allowed_payloads: list[OrderPayload], + product_id: str, + expected_status: int, + limit: int, + id_retrieval: int | str, + token_back: bool, ) -> None: - product_backend._allowed_payloads = create_order_allowed_payloads - - # check empty - res = stapi_client.get("/orders") + order_ids = prepare_order_pagination - default_orders = {"type": "FeatureCollection", "features": [], "links": []} - - assert res.status_code == status.HTTP_200_OK - assert res.headers["Content-Type"] == "application/geo+json" - assert res.json() == default_orders - - # add order to product - res = stapi_client.post( - f"products/{product_id}/orders", - json=create_order_allowed_payloads[0].model_dump(), + res = stapi_client.get( + "/orders", params={"next": order_ids[id_retrieval], "limit": limit} ) + assert res.status_code == expected_status - assert res.status_code == status.HTTP_201_CREATED, res.text - assert res.headers["Content-Type"] == "application/geo+json" - - res = stapi_client.post( - f"products/{product_id}/orders", - json=create_second_order_allowed_payloads[0].model_dump(), - ) - # call all orders - next = res.json()["id"] - res = stapi_client.get("/orders", params={"next": next, "limit": 1}) - checker = res.json() - - assert res.status_code == status.HTTP_200_OK - - # temp check to make sure token link isn't added to inside collection - for link in checker["features"][0]["links"]: + body = res.json() + for link in body["features"][0]["links"]: assert link["rel"] != "next" - assert checker["links"] != [] + assert body["links"] != [] # check to make sure new token in link - assert next not in checker["links"][0]["href"] + if token_back: + assert order_ids[id_retrieval] not in body["links"][0]["href"] + + assert len(body["features"]) == limit + - assert len(checker["features"]) == 1 +# test cases to check +# 1. Input token and get last record. Should not return a token if we are returning the last record - 'last' record being what is sorted +# 2. Input a crzy high limit - how to handle? Default to max or all records if less than max +# 3. Input token and get some intermediate records - return a token for next records +# 4. handle requesting an orderid/token that does't exist and returns 400/404. Bad token --> bad request. From 7b3453e64e4b6d274ae64dbf58472f30e016e56c Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Wed, 8 Jan 2025 13:24:47 -0500 Subject: [PATCH 09/35] tests: tweak test name --- tests/test_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_order.py b/tests/test_order.py index 0de5c5d..7461d77 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -263,7 +263,7 @@ def prepare_order_pagination( ), ], ) -def test_order_pagination_hold( +def test_order_pagination( prepare_order_pagination, stapi_client: TestClient, product_id: str, From 5db53c15703c9afe8de6551933b6a46227566705 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Wed, 8 Jan 2025 16:53:15 -0500 Subject: [PATCH 10/35] tests: reworked pagination test to iterate using next tokens to keep getting records until no more are available ftests: added separate test to get 404 back with no orders and using a token --- tests/application.py | 2 +- tests/test_order.py | 109 ++++++++++++------------------------------- 2 files changed, 31 insertions(+), 80 deletions(-) diff --git a/tests/application.py b/tests/application.py index 917260d..954a1eb 100644 --- a/tests/application.py +++ b/tests/application.py @@ -55,7 +55,7 @@ async def get_orders( limit = 100 order_ids = [*self._orders_db._orders.keys()] - if not order_ids: # no data in db + if not order_ids and not next: # no data in db return Success( ( OrderCollection( diff --git a/tests/test_order.py b/tests/test_order.py index 7461d77..29e29e0 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -197,19 +197,17 @@ def create_order_payloads() -> list[OrderPayload]: @pytest.fixture def prepare_order_pagination( stapi_client: TestClient, create_order_payloads: list[OrderPayload] -) -> tuple[str, str, str]: - # product_backend._allowed_payloads = create_order_payloads +) -> None: product_id = "test-spotlight" - # # check empty - # res = stapi_client.get("/orders") - # default_orders = {"type": "FeatureCollection", "features": [], "links": []} - # assert res.status_code == status.HTTP_200_OK - # assert res.headers["Content-Type"] == "application/geo+json" - # assert res.json() == default_orders + # check empty + res = stapi_client.get("/orders") + default_orders = {"type": "FeatureCollection", "features": [], "links": []} + assert res.status_code == status.HTTP_200_OK + assert res.headers["Content-Type"] == "application/geo+json" + assert res.json() == default_orders # get uuids created to use as pagination tokens - order_ids = [] for payload in create_order_payloads: res = stapi_client.post( f"products/{product_id}/orders", @@ -217,82 +215,35 @@ def prepare_order_pagination( ) assert res.status_code == status.HTTP_201_CREATED, res.text assert res.headers["Content-Type"] == "application/geo+json" - order_ids.append(res.json()["id"]) - - # res = stapi_client.get("/orders") - # checker = res.json() - # assert len(checker['features']) == 3 - return tuple(order_ids) + res = stapi_client.get("/orders") + checker = res.json() + assert len(checker["features"]) == 3 -@pytest.mark.parametrize( - "product_id,expected_status,limit,id_retrieval,token_back", - [ - pytest.param( - "test-spotlight", - status.HTTP_200_OK, - 1, - 0, - True, - id="input frst order_id token get new token back", - ), - pytest.param( - "test-spotlight", - status.HTTP_200_OK, - 1, - 2, - False, - id="input last order_id token get NO token back", - ), - pytest.param( - "test-spotlight", - status.HTTP_404_NOT_FOUND, - 1, - "BAD_TOKEN", - False, - id="input bad token get 404 back", - ), - pytest.param( - "test-spotlight", - status.HTTP_200_OK, - 1, - 1000000, - False, - id="high limit handled and returns valid records", - ), - ], -) def test_order_pagination( prepare_order_pagination, stapi_client: TestClient, - product_id: str, - expected_status: int, - limit: int, - id_retrieval: int | str, - token_back: bool, ) -> None: - order_ids = prepare_order_pagination - - res = stapi_client.get( - "/orders", params={"next": order_ids[id_retrieval], "limit": limit} - ) - assert res.status_code == expected_status + # prepare_order_pagination + res = stapi_client.get("/orders", params={"next": None, "limit": 1}) + assert res.status_code == status.HTTP_200_OK body = res.json() - for link in body["features"][0]["links"]: - assert link["rel"] != "next" - assert body["links"] != [] - - # check to make sure new token in link - if token_back: - assert order_ids[id_retrieval] not in body["links"][0]["href"] - - assert len(body["features"]) == limit - - -# test cases to check -# 1. Input token and get last record. Should not return a token if we are returning the last record - 'last' record being what is sorted -# 2. Input a crzy high limit - how to handle? Default to max or all records if less than max -# 3. Input token and get some intermediate records - return a token for next records -# 4. handle requesting an orderid/token that does't exist and returns 400/404. Bad token --> bad request. + next = body["links"][0]["href"] + + while next: + res = stapi_client.get(next) + assert res.status_code == status.HTTP_200_OK + body = res.json() + if body["links"]: + next = body["links"][0]["href"] + else: + break + + +# separate test here to check for bad token getting back 404 +def test_token_not_found(stapi_client: TestClient): + res = stapi_client.get("/orders", params={"next": "a_token"}) + # should return 404 as a result of bad token + assert res.status_code == status.HTTP_404_NOT_FOUND From b690cf16316f90e6115bd79f4cc4f2c0bd1bb09f Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 9 Jan 2025 10:29:38 -0500 Subject: [PATCH 11/35] feat: tweaking mock backend to use list index look up and return empty features after token lookup tests: separating empty orders check to separate test --- src/stapi_fastapi/routers/root_router.py | 2 +- tests/application.py | 14 ++++---------- tests/test_order.py | 14 +++++++------- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 0e8753a..7929872 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -184,7 +184,7 @@ async def get_orders( return collections case Failure(e): logging.exception(f"An error occurred while retrieving orders: {e}") - if isinstance(e, IndexError): + if isinstance(e, ValueError): raise NotFoundException(detail="Error finding pagination token") else: raise HTTPException( diff --git a/tests/application.py b/tests/application.py index 954a1eb..5474dcd 100644 --- a/tests/application.py +++ b/tests/application.py @@ -55,18 +55,12 @@ async def get_orders( limit = 100 order_ids = [*self._orders_db._orders.keys()] - if not order_ids and not next: # no data in db - return Success( - ( - OrderCollection( - features=list(self._orders_db._orders.values()) - ), - "", - ) - ) if next: - start = [i for i, x in enumerate(order_ids) if x == next][0] + start = order_ids.index(next) + if not order_ids and not next: # no data in db + return Success((OrderCollection(features=[]), "")) + end = start + limit ids = order_ids[start:end] feats = [self._orders_db._orders[order_id] for order_id in ids] diff --git a/tests/test_order.py b/tests/test_order.py index 29e29e0..0363f27 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -194,19 +194,19 @@ def create_order_payloads() -> list[OrderPayload]: return payloads -@pytest.fixture -def prepare_order_pagination( - stapi_client: TestClient, create_order_payloads: list[OrderPayload] -) -> None: - product_id = "test-spotlight" - - # check empty +def test_empty_order(stapi_client: TestClient): res = stapi_client.get("/orders") default_orders = {"type": "FeatureCollection", "features": [], "links": []} assert res.status_code == status.HTTP_200_OK assert res.headers["Content-Type"] == "application/geo+json" assert res.json() == default_orders + +@pytest.fixture +def prepare_order_pagination( + stapi_client: TestClient, create_order_payloads: list[OrderPayload] +) -> None: + product_id = "test-spotlight" # get uuids created to use as pagination tokens for payload in create_order_payloads: res = stapi_client.post( From 55e1c6018a918e8b9d8fdac14576a8997dcbfb15 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 9 Jan 2025 11:40:15 -0500 Subject: [PATCH 12/35] tests: moving tests around to work around issue of stapi_client fixture not being torn down for every test --- tests/test_order.py | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/tests/test_order.py b/tests/test_order.py index c36dc6f..f1c7b84 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -18,6 +18,14 @@ END = START + timedelta(days=5) +def test_empty_order(stapi_client: TestClient): + res = stapi_client.get("/orders") + default_orders = {"type": "FeatureCollection", "features": [], "links": []} + assert res.status_code == status.HTTP_200_OK + assert res.headers["Content-Type"] == "application/geo+json" + assert res.json() == default_orders + + @pytest.fixture def create_order_allowed_payloads() -> list[OrderPayload]: return [ @@ -194,33 +202,6 @@ def create_order_payloads() -> list[OrderPayload]: return payloads -def test_empty_order(stapi_client: TestClient): - res = stapi_client.get("/orders") - default_orders = {"type": "FeatureCollection", "features": [], "links": []} - assert res.status_code == status.HTTP_200_OK - assert res.headers["Content-Type"] == "application/geo+json" - assert res.json() == default_orders - - -@pytest.fixture -def prepare_order_pagination( - stapi_client: TestClient, create_order_payloads: list[OrderPayload] -) -> None: - product_id = "test-spotlight" - # get uuids created to use as pagination tokens - for payload in create_order_payloads: - res = stapi_client.post( - f"products/{product_id}/orders", - json=payload.model_dump(), - ) - assert res.status_code == status.HTTP_201_CREATED, res.text - assert res.headers["Content-Type"] == "application/geo+json" - - res = stapi_client.get("/orders") - checker = res.json() - assert len(checker["features"]) == 3 - - def test_order_pagination( prepare_order_pagination, stapi_client: TestClient, @@ -247,4 +228,3 @@ def test_token_not_found(stapi_client: TestClient): res = stapi_client.get("/orders", params={"next": "a_token"}) # should return 404 as a result of bad token assert res.status_code == status.HTTP_404_NOT_FOUND - From 8886988304c6fb053f18ef5ba78cbc7df7505dff Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 9 Jan 2025 12:27:17 -0500 Subject: [PATCH 13/35] feat: clean up root backend model after messy main merge tests: fixed issue with in memory DB by adding missing __init__ to class. tests: added multiple orders to create_order fixture and passed this fixture to other tests using previously existing create_order_allowed_paylaods tests: added setup_pagination fixture that is now necessary again after fixing the in memory db init issue --- src/stapi_fastapi/backends/root_backend.py | 2 +- tests/application.py | 5 +- tests/test_order.py | 111 ++++++--------------- 3 files changed, 36 insertions(+), 82 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index a0f574b..9259baf 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -11,7 +11,7 @@ ) -class RootBackend[T: OrderStatusPayload, U: OrderStatus](Protocol): # pragma: nocover +class RootBackend[T: OrderStatus](Protocol): # pragma: nocover async def get_orders( self, request: Request, next: str | None, limit: int ) -> ResultE[tuple[OrderCollection, str]]: diff --git a/tests/application.py b/tests/application.py index 95c1ce9..5c824af 100644 --- a/tests/application.py +++ b/tests/application.py @@ -34,8 +34,9 @@ class InMemoryOrderDB: - _orders: dict[str, Order] = {} - _statuses: dict[str, list[OrderStatus]] = defaultdict(list) + def __init__(self) -> None: + self._orders: dict[str, Order] = {} + self._statuses: dict[str, list[OrderStatus]] = defaultdict(list) class MockRootBackend(RootBackend): diff --git a/tests/test_order.py b/tests/test_order.py index f1c7b84..71557ef 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -27,20 +27,27 @@ def test_empty_order(stapi_client: TestClient): @pytest.fixture -def create_order_allowed_payloads() -> list[OrderPayload]: - return [ - OrderPayload( +def create_order_payloads() -> list[OrderPayload]: + datetimes = [ + ("2024-10-09T18:55:33Z", "2024-10-12T18:55:33Z"), + ("2024-10-15T18:55:33Z", "2024-10-18T18:55:33Z"), + ("2024-10-20T18:55:33Z", "2024-10-23T18:55:33Z"), + ] + payloads = [] + for start, end in datetimes: + payload = OrderPayload( geometry=Point( - type="Point", coordinates=Position2D(longitude=13.4, latitude=52.5) + type="Point", coordinates=Position2D(longitude=14.4, latitude=56.5) ), datetime=( - datetime.fromisoformat("2024-11-11T18:55:33Z"), - datetime.fromisoformat("2024-11-15T18:55:33Z"), + datetime.fromisoformat(start), + datetime.fromisoformat(end), ), filter=None, order_parameters=MyOrderParameters(s3_path="s3://my-bucket"), - ), - ] + ) + payloads.append(payload) + return payloads @pytest.fixture @@ -48,13 +55,12 @@ def new_order_response( product_id: str, product_backend: MockProductBackend, stapi_client: TestClient, - create_order_allowed_payloads: list[OrderPayload], + create_order_payloads: list[OrderPayload], ) -> Response: - product_backend._allowed_payloads = create_order_allowed_payloads - + product_backend._allowed_payloads = create_order_payloads res = stapi_client.post( f"products/{product_id}/orders", - json=create_order_allowed_payloads[0].model_dump(), + json=create_order_payloads[0].model_dump(), ) assert res.status_code == status.HTTP_201_CREATED, res.text @@ -105,23 +111,23 @@ def get_order_response( @pytest.mark.parametrize("product_id", ["test-spotlight"]) def test_get_order_properties( - get_order_response: Response, create_order_allowed_payloads + get_order_response: Response, create_order_payloads ) -> None: order = get_order_response.json() assert order["geometry"] == { "type": "Point", - "coordinates": list(create_order_allowed_payloads[0].geometry.coordinates), + "coordinates": list(create_order_payloads[0].geometry.coordinates), } assert order["properties"]["search_parameters"]["geometry"] == { "type": "Point", - "coordinates": list(create_order_allowed_payloads[0].geometry.coordinates), + "coordinates": list(create_order_payloads[0].geometry.coordinates), } assert ( order["properties"]["search_parameters"]["datetime"] - == create_order_allowed_payloads[0].model_dump()["datetime"] + == create_order_payloads[0].model_dump()["datetime"] ) @@ -141,73 +147,21 @@ def test_order_status_after_create( assert len(res.json()["statuses"]) == 1 -@pytest.mark.parametrize("product_id", ["test-spotlight"]) -def test_order_status_after_update( - get_order_response: Response, stapi_client: TestClient -) -> None: - body = get_order_response.json() - statuses_url = find_link(body["links"], "monitor")["href"] - - res = stapi_client.post( - statuses_url, - json={ - "status_code": "accepted", - "reason_code": "REASON1", - "reason_text": "some reason", - }, - ) - - assert res.status_code == status.HTTP_202_ACCEPTED - - res = stapi_client.get(statuses_url) - assert res.status_code == status.HTTP_200_OK - assert res.headers["Content-Type"] == "application/json" - body = res.json() - assert len(body["statuses"]) == 2 - - s = body["statuses"][0] - assert s["reason_code"] == "REASON1" - assert s["reason_text"] == "some reason" - assert s["status_code"] == "accepted" - assert s["timestamp"] - - s = body["statuses"][1] - assert s["reason_code"] is None - assert s["reason_text"] is None - assert s["status_code"] == "received" - assert s["timestamp"] - - @pytest.fixture -def create_order_payloads() -> list[OrderPayload]: - datetimes = [ - ("2024-10-09T18:55:33Z", "2024-10-12T18:55:33Z"), - ("2024-10-15T18:55:33Z", "2024-10-18T18:55:33Z"), - ("2024-10-20T18:55:33Z", "2024-10-23T18:55:33Z"), - ] - payloads = [] - for start, end in datetimes: - payload = OrderPayload( - geometry=Point( - type="Point", coordinates=Position2D(longitude=14.4, latitude=56.5) - ), - datetime=( - datetime.fromisoformat(start), - datetime.fromisoformat(end), - ), - filter=None, - order_parameters=MyOrderParameters(s3_path="s3://my-bucket"), +def setup_pagination(stapi_client: TestClient, create_order_payloads) -> None: + product_id = "test-spotlight" + + for order in create_order_payloads: + res = stapi_client.post( + f"products/{product_id}/orders", + json=order.model_dump(), ) - payloads.append(payload) - return payloads + assert res.status_code == status.HTTP_201_CREATED, res.text + assert res.headers["Content-Type"] == "application/geo+json" -def test_order_pagination( - prepare_order_pagination, - stapi_client: TestClient, -) -> None: - # prepare_order_pagination +def test_order_pagination(stapi_client: TestClient, setup_pagination) -> None: res = stapi_client.get("/orders", params={"next": None, "limit": 1}) assert res.status_code == status.HTTP_200_OK body = res.json() @@ -223,7 +177,6 @@ def test_order_pagination( break -# separate test here to check for bad token getting back 404 def test_token_not_found(stapi_client: TestClient): res = stapi_client.get("/orders", params={"next": "a_token"}) # should return 404 as a result of bad token From 2fca5c79dbc474951b18bdc3926f6ce6067fbc4b Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 9 Jan 2025 12:48:44 -0500 Subject: [PATCH 14/35] tests: add limit check assertion in pagination test --- tests/test_order.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_order.py b/tests/test_order.py index 71557ef..1957475 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -161,8 +161,9 @@ def setup_pagination(stapi_client: TestClient, create_order_payloads) -> None: assert res.headers["Content-Type"] == "application/geo+json" -def test_order_pagination(stapi_client: TestClient, setup_pagination) -> None: - res = stapi_client.get("/orders", params={"next": None, "limit": 1}) +@pytest.mark.parametrize("limit", [2]) +def test_order_pagination(stapi_client: TestClient, setup_pagination, limit) -> None: + res = stapi_client.get("/orders", params={"next": None, "limit": limit}) assert res.status_code == status.HTTP_200_OK body = res.json() next = body["links"][0]["href"] @@ -172,6 +173,7 @@ def test_order_pagination(stapi_client: TestClient, setup_pagination) -> None: assert res.status_code == status.HTTP_200_OK body = res.json() if body["links"]: + assert len(body["features"]) == limit next = body["links"][0]["href"] else: break From d3c5aa8dfff341de1c49637daf11e9b818aede0d Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 10 Jan 2025 13:25:42 -0500 Subject: [PATCH 15/35] feat: adding pagination to GET /products endpoint in root_router.py tests: adding tests for GET /products pagination --- src/stapi_fastapi/routers/root_router.py | 44 ++++++++++++++++++++---- tests/conftest.py | 36 ++++++++++++++++++- tests/test_order.py | 4 +-- tests/test_product.py | 33 ++++++++++++++++++ 4 files changed, 107 insertions(+), 10 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index f9ae395..91228ef 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -140,17 +140,47 @@ def get_root(self, request: Request) -> RootResponse: def get_conformance(self, request: Request) -> Conformance: return Conformance(conforms_to=self.conformances) - def get_products(self, request: Request) -> ProductsCollection: - return ProductsCollection( - products=[pr.get_product(request) for pr in self.product_routers.values()], - links=[ + def get_products( + self, request: Request, next: str | None = None, limit: int = 10 + ) -> ProductsCollection: + try: + start = 0 + product_ids = [*self.product_routers.keys()] + if next: + start = product_ids.index(next) + if not product_ids and not next: + ProductsCollection( + products=[], + links=[ + Link( + href=str(request.url_for(f"{self.name}:list-products")), + rel="self", + type=TYPE_JSON, + ) + ], + ) + end = start + limit + ids = product_ids[start:end] + products = [ + self.product_routers[product_id].get_product(request) + for product_id in ids + ] + links = [ Link( href=str(request.url_for(f"{self.name}:list-products")), rel="self", type=TYPE_JSON, - ) - ], - ) + ), + ] + next = "" + if end < len(product_ids): + next = self.product_routers[product_ids[end]].product.id + updated_url = request.url.include_query_params(next=next) + links.append(Link(href=str(updated_url), rel="next", type=TYPE_JSON)) + return ProductsCollection(products=products, links=links) + except ValueError as e: + logging.exception(f"An error occurred while retrieving orders: {e}") + raise NotFoundException(detail="Error finding pagination token") async def get_orders( self, request: Request, next: str | None = None, limit: int = 10 diff --git a/tests/conftest.py b/tests/conftest.py index 82eb841..9b583bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,12 +71,46 @@ def mock_product_test_spotlight( ) +@pytest.fixture +def mock_product_test_wolf_cola( + product_backend: MockProductBackend, mock_provider: Provider +) -> Product: + """Fixture for a mock Wolf Cola product.""" + return Product( + id="test-wolf-cola", + title="Test Wolf Cola Product", + description="Test product for Wolf Cola for testing GET /product pagination", + license="CC-BY-4.0", + keywords=["test", "satellite", "wolf-cola"], + providers=[mock_provider], + links=[], + constraints=MyProductConstraints, + opportunity_properties=MyOpportunityProperties, + order_parameters=MyOrderParameters, + backend=product_backend, + ) + + @pytest.fixture def stapi_client( - root_backend, mock_product_test_spotlight, base_url: str + root_backend, + mock_product_test_spotlight, + mock_product_test_wolf_cola, + base_url: str, ) -> Iterator[TestClient]: root_router = RootRouter(root_backend) root_router.add_product(mock_product_test_spotlight) + root_router.add_product(mock_product_test_wolf_cola) + app = FastAPI() + app.include_router(root_router, prefix="") + + with TestClient(app, base_url=f"{base_url}") as client: + yield client + + +@pytest.fixture +def empty_stapi_client(root_backend, base_url: str) -> Iterator[TestClient]: + root_router = RootRouter(root_backend) app = FastAPI() app.include_router(root_router, prefix="") diff --git a/tests/test_order.py b/tests/test_order.py index 1957475..72d2f41 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -166,6 +166,7 @@ def test_order_pagination(stapi_client: TestClient, setup_pagination, limit) -> res = stapi_client.get("/orders", params={"next": None, "limit": limit}) assert res.status_code == status.HTTP_200_OK body = res.json() + assert len(body["features"]) == limit next = body["links"][0]["href"] while next: @@ -179,7 +180,6 @@ def test_order_pagination(stapi_client: TestClient, setup_pagination, limit) -> break -def test_token_not_found(stapi_client: TestClient): +def test_token_not_found(stapi_client: TestClient) -> None: res = stapi_client.get("/orders", params={"next": "a_token"}) - # should return 404 as a result of bad token assert res.status_code == status.HTTP_404_NOT_FOUND diff --git a/tests/test_product.py b/tests/test_product.py index ac8e0cf..b10b866 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -63,3 +63,36 @@ def test_product_order_parameters_response( json_schema = res.json() assert "properties" in json_schema assert "s3_path" in json_schema["properties"] + + +@pytest.mark.parametrize("limit", [1]) +def test_product_pagination(stapi_client: TestClient, limit: int): + res = stapi_client.get("/products", params={"next": None, "limit": limit}) + assert res.status_code == status.HTTP_200_OK + body = res.json() + assert len(body["products"]) == limit + for d in body["links"]: + if ("rel", "next") in d.items(): + next = d["href"] + + while len(body["links"]) > 1: + res = stapi_client.get(next) + assert res.status_code == status.HTTP_200_OK + body = res.json() + assert len(body["products"]) == limit + for d in body["links"]: + if ("rel", "next") in d.items(): + next = body["links"][0]["href"] + + +def test_token_not_found(stapi_client: TestClient) -> None: + res = stapi_client.get("/products", params={"next": "a_token"}) + assert res.status_code == status.HTTP_404_NOT_FOUND + + +def test_no_products(empty_stapi_client: TestClient): + res = empty_stapi_client.get("/products") + body = res.json() + print("hold") + assert res.status_code == status.HTTP_200_OK + assert len(body["products"]) == 0 From ac661d9a1d7237123fa4bad4773e737de8594dc8 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 10 Jan 2025 17:02:28 -0500 Subject: [PATCH 16/35] feat: adding pagination for GET orders/order_id/statuses inputs to root_router and fleshing out mock root backend --- src/stapi_fastapi/backends/root_backend.py | 4 ++-- src/stapi_fastapi/routers/product_router.py | 6 ++++- src/stapi_fastapi/routers/root_router.py | 8 +++++-- tests/application.py | 26 ++++++++++++++++++--- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 9259baf..5b1dc41 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -35,8 +35,8 @@ async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Orde ... async def get_order_statuses( - self, order_id: str, request: Request - ) -> ResultE[list[T]]: + self, order_id: str, request: Request, next: str | None, limit: int + ) -> ResultE[tuple[list[T], str]]: """ Get statuses for order with `order_id`. diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index e98738c..43de3f0 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -47,6 +47,10 @@ def __init__( tags=["Products"], ) + # Any paginated POST or requires a body needs the same POST body to be sent + # method property on Link object needs to be set to POST + # next link for GET can just include next token dont need to specify method or body + # where there's a POST body, have to include post body body property is request.body() self.add_api_route( path="/opportunities", endpoint=self.search_opportunities, @@ -171,7 +175,7 @@ async def search_opportunities( return OpportunityCollection( features=features, links=[ - Link( + Link( # current bug is missing method set and setting body for href=str( request.url_for( f"{self.root_router.name}:{self.product.id}:create-order", diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 91228ef..d10d5cb 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -241,9 +241,13 @@ async def get_order(self: Self, order_id: str, request: Request) -> Order: raise AssertionError("Expected code to be unreachable") async def get_order_statuses( - self: Self, order_id: str, request: Request + self: Self, + order_id: str, + request: Request, + next: str | None = None, + limit: int = 10, ) -> OrderStatuses: - match await self.backend.get_order_statuses(order_id, request): + match await self.backend.get_order_statuses(order_id, request, next, limit): case Success(statuses): return OrderStatuses( statuses=statuses, diff --git a/tests/application.py b/tests/application.py index 5c824af..a2d59ef 100644 --- a/tests/application.py +++ b/tests/application.py @@ -80,9 +80,29 @@ async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Orde return Success(Maybe.from_optional(self._orders_db._orders.get(order_id))) async def get_order_statuses( - self, order_id: str, request: Request - ) -> ResultE[list[OrderStatus]]: - return Success(self._orders_db._statuses[order_id]) + self, order_id: str, request: Request, next: str | None, limit: int + ) -> ResultE[tuple[list[OrderStatus], str]]: + try: + start = 0 + if limit > 100: + limit = 100 + statuses = self._orders_db._statuses[order_id] + + if next: + start = int(next) + if not statuses and not next: + return Success(([], "")) + + end = start + limit + stati = statuses[start:end] + + next = "" + if end < len(statuses): + next = str(end) + return Success((stati, next)) + # return Success(self._orders_db._statuses[order_id]) + except Exception as e: + return Failure(e) class MockProductBackend(ProductBackend): From 690e44fe059aab241defd35ae9520e37234d173c Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Tue, 14 Jan 2025 11:06:35 -0500 Subject: [PATCH 17/35] feat: refactoring get_products pagination based on feedback, refactoring get_orders pagination based on feedback, feat: adding pagination for get_order_statuses feat: updating mock backend for get_order_statuses pagination tests: adding test for get order status pagination tests: adding tests for order_satuses paginatio --- src/stapi_fastapi/backends/root_backend.py | 3 +- src/stapi_fastapi/routers/root_router.py | 135 ++++++++++++--------- tests/application.py | 47 ++++--- tests/conftest.py | 23 ++++ tests/test_order.py | 43 ++++++- 5 files changed, 174 insertions(+), 77 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 5b1dc41..427cbf0 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -6,7 +6,6 @@ from stapi_fastapi.models.order import ( Order, - OrderCollection, OrderStatus, ) @@ -14,7 +13,7 @@ class RootBackend[T: OrderStatus](Protocol): # pragma: nocover async def get_orders( self, request: Request, next: str | None, limit: int - ) -> ResultE[tuple[OrderCollection, str]]: + ) -> ResultE[tuple[list[Order], str]]: """ Return a list of existing orders and pagination token if applicable No pagination will return empty string for token diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index d10d5cb..0ae2d4e 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -143,51 +143,50 @@ def get_conformance(self, request: Request) -> Conformance: def get_products( self, request: Request, next: str | None = None, limit: int = 10 ) -> ProductsCollection: + start = 0 + product_ids = [*self.product_routers.keys()] + end = min(start + limit, len(product_ids)) try: - start = 0 - product_ids = [*self.product_routers.keys()] if next: start = product_ids.index(next) - if not product_ids and not next: - ProductsCollection( - products=[], - links=[ - Link( - href=str(request.url_for(f"{self.name}:list-products")), - rel="self", - type=TYPE_JSON, - ) - ], + except ValueError as e: + logging.exception("An error occurred while retrieving orders") + raise NotFoundException(detail="Error finding pagination token") from e + + ids = product_ids[start:end] + links = [ + Link( + href=str(request.url_for(f"{self.name}:list-products")), + rel="self", + type=TYPE_JSON, + ), + ] + if end < len(product_ids): + links.append( + Link( + href=str( + request.url.include_query_params( + next=self.product_routers[product_ids[end]].product.id + ), + ), + rel="next", + type=TYPE_JSON, ) - end = start + limit - ids = product_ids[start:end] - products = [ + ) + return ProductsCollection( + products=[ self.product_routers[product_id].get_product(request) for product_id in ids - ] - links = [ - Link( - href=str(request.url_for(f"{self.name}:list-products")), - rel="self", - type=TYPE_JSON, - ), - ] - next = "" - if end < len(product_ids): - next = self.product_routers[product_ids[end]].product.id - updated_url = request.url.include_query_params(next=next) - links.append(Link(href=str(updated_url), rel="next", type=TYPE_JSON)) - return ProductsCollection(products=products, links=links) - except ValueError as e: - logging.exception(f"An error occurred while retrieving orders: {e}") - raise NotFoundException(detail="Error finding pagination token") + ], + links=links, + ) async def get_orders( self, request: Request, next: str | None = None, limit: int = 10 ) -> OrderCollection: match await self.backend.get_orders(request, next, limit): - case Success((collections, token)): - for order in collections: + case Success((orders, pagination_token)): + for order in orders: order.links.append( Link( href=str( @@ -199,14 +198,27 @@ async def get_orders( type=TYPE_JSON, ) ) - if token: # pagination link if backend returns token - updated_url = request.url.include_query_params(next=token) - collections.links.append( - Link(href=str(updated_url), rel="next", type=TYPE_JSON) + if pagination_token: + return OrderCollection( + features=orders, + links=[ + Link( + href=str( + request.url.include_query_params( + next=pagination_token + ) + ), + rel="next", + type=TYPE_JSON, + ) + ], ) - return collections + return OrderCollection(features=orders) case Failure(e): - logging.exception(f"An error occurred while retrieving orders: {e}") + logger.error( + "An error occurred while retrieving orders: %s", + traceback.format_exception(e), + ) if isinstance(e, ValueError): raise NotFoundException(detail="Error finding pagination token") else: @@ -248,31 +260,46 @@ async def get_order_statuses( limit: int = 10, ) -> OrderStatuses: match await self.backend.get_order_statuses(order_id, request, next, limit): - case Success(statuses): - return OrderStatuses( - statuses=statuses, - links=[ + case Success((statuses, pagination_token)): + links = [ + Link( + href=str( + request.url_for( + f"{self.name}:list-order-statuses", + order_id=order_id, + ) + ), + rel="self", + type=TYPE_JSON, + ) + ] + if pagination_token: + links.append( Link( href=str( - request.url_for( - f"{self.name}:list-order-statuses", - order_id=order_id, - ) + request.url.include_query_params(next=pagination_token) ), - rel="self", + rel="next", type=TYPE_JSON, ) - ], + ) + return OrderStatuses(statuses=statuses, links=links) + return OrderStatuses( + statuses=statuses, + links=links, ) case Failure(e): logger.error( "An error occurred while retrieving order statuses: %s", traceback.format_exception(e), ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error finding Order Statuses", - ) + if isinstance(e, KeyError): + raise NotFoundException(detail="Error finding pagination token") + else: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error finding Order Statuses", + ) case _: raise AssertionError("Expected code to be unreachable") diff --git a/tests/application.py b/tests/application.py index a2d59ef..87922c3 100644 --- a/tests/application.py +++ b/tests/application.py @@ -18,7 +18,6 @@ ) from stapi_fastapi.models.order import ( Order, - OrderCollection, OrderParameters, OrderPayload, OrderStatus, @@ -45,7 +44,7 @@ def __init__(self, orders: InMemoryOrderDB) -> None: async def get_orders( self, request: Request, next: str | None, limit: int - ) -> ResultE[tuple[OrderCollection, str]]: + ) -> ResultE[tuple[list[Order], str]]: """ Return orders from backend. Handle pagination/limit if applicable """ @@ -53,22 +52,17 @@ async def get_orders( start = 0 if limit > 100: limit = 100 - order_ids = [*self._orders_db._orders.keys()] if next: start = order_ids.index(next) - if not order_ids and not next: # no data in db - return Success((OrderCollection(features=[]), "")) - - end = start + limit + end = min(start + limit, len(order_ids)) ids = order_ids[start:end] - feats = [self._orders_db._orders[order_id] for order_id in ids] + orders = [self._orders_db._orders[order_id] for order_id in ids] - next = "" if end < len(order_ids): - next = self._orders_db._orders[order_ids[end]].id - return Success((OrderCollection(features=feats), next)) + return Success((orders, self._orders_db._orders[order_ids[end]].id)) + return Success((orders, "")) except Exception as e: return Failure(e) @@ -90,17 +84,12 @@ async def get_order_statuses( if next: start = int(next) - if not statuses and not next: - return Success(([], "")) - - end = start + limit + end = min(start + limit, len(statuses)) stati = statuses[start:end] - next = "" if end < len(statuses): - next = str(end) - return Success((stati, next)) - # return Success(self._orders_db._statuses[order_id]) + return Success((stati, str(end))) + return Success((stati, "")) except Exception as e: return Failure(e) @@ -217,3 +206,23 @@ class MyOrderParameters(OrderParameters): root_router.add_product(product) app: FastAPI = FastAPI() app.include_router(root_router, prefix="") + +TEST_STATUSES = { + "test_order_id": [ + OrderStatus( + timestamp=datetime(2025, 1, 14, 2, 21, 48, 466726, tzinfo=timezone.utc), + status_code=OrderStatusCode.received, + links=[], + ), + OrderStatus( + timestamp=datetime(2025, 1, 15, 5, 20, 48, 466726, tzinfo=timezone.utc), + status_code=OrderStatusCode.accepted, + links=[], + ), + OrderStatus( + timestamp=datetime(2025, 1, 16, 10, 15, 32, 466726, tzinfo=timezone.utc), + status_code=OrderStatusCode.completed, + links=[], + ), + ] +} diff --git a/tests/conftest.py b/tests/conftest.py index 9b583bd..49f40b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ from stapi_fastapi.routers.root_router import RootRouter from .application import ( + TEST_STATUSES, InMemoryOrderDB, MockProductBackend, MockRootBackend, @@ -41,6 +42,13 @@ def order_db() -> InMemoryOrderDB: return InMemoryOrderDB() +@pytest.fixture +def order_db_statuses() -> InMemoryOrderDB: + order_db = InMemoryOrderDB() + order_db._statuses = TEST_STATUSES + return order_db + + @pytest.fixture def product_backend(order_db: InMemoryOrderDB) -> MockProductBackend: return MockProductBackend(order_db) @@ -51,6 +59,11 @@ def root_backend(order_db: InMemoryOrderDB) -> MockRootBackend: return MockRootBackend(order_db) +@pytest.fixture +def root_backend_preloaded(order_db_statuses: InMemoryOrderDB) -> MockRootBackend: + return MockRootBackend(order_db_statuses) + + @pytest.fixture def mock_product_test_spotlight( product_backend: MockProductBackend, mock_provider: Provider @@ -118,6 +131,16 @@ def empty_stapi_client(root_backend, base_url: str) -> Iterator[TestClient]: yield client +@pytest.fixture +def statuses_client(root_backend_preloaded, base_url: str) -> Iterator[TestClient]: + root_router = RootRouter(root_backend_preloaded) + app = FastAPI() + app.include_router(root_router, prefix="") + + with TestClient(app, base_url=f"{base_url}") as client: + yield client + + @pytest.fixture(scope="session") def url_for(base_url: str) -> Iterator[Callable[[str], str]]: def with_trailing_slash(value: str) -> str: diff --git a/tests/test_order.py b/tests/test_order.py index 72d2f41..4b935b5 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -148,7 +148,7 @@ def test_order_status_after_create( @pytest.fixture -def setup_pagination(stapi_client: TestClient, create_order_payloads) -> None: +def setup_orders_pagination(stapi_client: TestClient, create_order_payloads) -> None: product_id = "test-spotlight" for order in create_order_payloads: @@ -162,7 +162,9 @@ def setup_pagination(stapi_client: TestClient, create_order_payloads) -> None: @pytest.mark.parametrize("limit", [2]) -def test_order_pagination(stapi_client: TestClient, setup_pagination, limit) -> None: +def test_order_pagination( + stapi_client: TestClient, setup_orders_pagination, limit +) -> None: res = stapi_client.get("/orders", params={"next": None, "limit": limit}) assert res.status_code == status.HTTP_200_OK body = res.json() @@ -173,6 +175,7 @@ def test_order_pagination(stapi_client: TestClient, setup_pagination, limit) -> res = stapi_client.get(next) assert res.status_code == status.HTTP_200_OK body = res.json() + assert body["features"] != [] if body["links"]: assert len(body["features"]) == limit next = body["links"][0]["href"] @@ -183,3 +186,39 @@ def test_order_pagination(stapi_client: TestClient, setup_pagination, limit) -> def test_token_not_found(stapi_client: TestClient) -> None: res = stapi_client.get("/orders", params={"next": "a_token"}) assert res.status_code == status.HTTP_404_NOT_FOUND + + +def test_order_status_pagination(statuses_client: TestClient, limit: int = 2) -> None: + order_id = "test_order_id" + res = statuses_client.get(f"/orders/{order_id}/statuses") + assert res.status_code == status.HTTP_200_OK + + order_id = "test_order_id" + res = statuses_client.get( + f"/orders/{order_id}/statuses", params={"next": None, "limit": limit} + ) + body = res.json() + links = body["links"] + for link in body["links"]: + if ("rel", "next") in link.items(): + assert len(body["statuses"]) == limit + next = link["href"] + + while len(links) > 1: + res = statuses_client.get(next) + assert res.status_code == status.HTTP_200_OK + body = res.json() + assert body["statuses"] != [] + links = body["links"] + for link in body["links"]: + if ("rel", "next") in link.items(): + assert len(body["statuses"]) == limit + next = body["links"][0]["href"] + + +def test_get_order_statuses_bad_token( + statuses_client: TestClient, limit: int = 2 +) -> None: + order_id = "non_existing_order_id" + res = statuses_client.get(f"/orders/{order_id}/statuses") + assert res.status_code == status.HTTP_404_NOT_FOUND From e562d56d63d357dbd8df228631ce55b80dbd4a62 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Tue, 14 Jan 2025 11:37:07 -0500 Subject: [PATCH 18/35] fix: remove unnecessar 'from e' in get_products() --- src/stapi_fastapi/routers/root_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 0ae2d4e..d9850b5 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -149,9 +149,9 @@ def get_products( try: if next: start = product_ids.index(next) - except ValueError as e: + except ValueError: logging.exception("An error occurred while retrieving orders") - raise NotFoundException(detail="Error finding pagination token") from e + raise NotFoundException(detail="Error finding pagination token") from None ids = product_ids[start:end] links = [ From 1cdc1457abb6bdf7cd83567ad18853b3e38c90ae Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Tue, 14 Jan 2025 12:13:37 -0500 Subject: [PATCH 19/35] fix: fix bug in get_products where end was not being propery calculated by being calculated before new start index --- src/stapi_fastapi/routers/root_router.py | 3 +-- tests/test_product.py | 7 +++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index d9850b5..ce6dce6 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -145,14 +145,13 @@ def get_products( ) -> ProductsCollection: start = 0 product_ids = [*self.product_routers.keys()] - end = min(start + limit, len(product_ids)) try: if next: start = product_ids.index(next) except ValueError: logging.exception("An error occurred while retrieving orders") raise NotFoundException(detail="Error finding pagination token") from None - + end = min(start + limit, len(product_ids)) ids = product_ids[start:end] links = [ Link( diff --git a/tests/test_product.py b/tests/test_product.py index b10b866..054c911 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -71,17 +71,20 @@ def test_product_pagination(stapi_client: TestClient, limit: int): assert res.status_code == status.HTTP_200_OK body = res.json() assert len(body["products"]) == limit + links = body["links"] for d in body["links"]: if ("rel", "next") in d.items(): next = d["href"] - while len(body["links"]) > 1: + while len(links) > 1: res = stapi_client.get(next) assert res.status_code == status.HTTP_200_OK body = res.json() - assert len(body["products"]) == limit + assert body["products"] != [] + links = body["links"] for d in body["links"]: if ("rel", "next") in d.items(): + assert len(body["products"]) == limit next = body["links"][0]["href"] From af2e2ade2b0f315621e798b8db9a80f4024609ce Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Tue, 14 Jan 2025 14:41:25 -0500 Subject: [PATCH 20/35] fix: load mock data into in memory db more cleanly for order status tests. --- tests/application.py | 40 ++++++++++++++----------------- tests/conftest.py | 24 ------------------- tests/test_order.py | 56 +++++++++++++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 55 deletions(-) diff --git a/tests/application.py b/tests/application.py index 87922c3..de1ef5f 100644 --- a/tests/application.py +++ b/tests/application.py @@ -105,10 +105,26 @@ async def search_opportunities( product_router: ProductRouter, search: OpportunityRequest, request: Request, - ) -> ResultE[list[Opportunity]]: + next: str | None, + limit: int, + ) -> ResultE[tuple[list[Opportunity], str]]: try: + start = 0 + if limit > 100: + limit = 100 + if next: + start = self._opportunities.index(next) + end = min(start + limit, len(self._opportunities)) + print(end) + # opportunities = self._opportunities.k return Success( - [o.model_copy(update=search.model_dump()) for o in self._opportunities] + ( + [ + o.model_copy(update=search.model_dump()) + for o in self._opportunities + ], + "", + ) ) except Exception as e: return Failure(e) @@ -206,23 +222,3 @@ class MyOrderParameters(OrderParameters): root_router.add_product(product) app: FastAPI = FastAPI() app.include_router(root_router, prefix="") - -TEST_STATUSES = { - "test_order_id": [ - OrderStatus( - timestamp=datetime(2025, 1, 14, 2, 21, 48, 466726, tzinfo=timezone.utc), - status_code=OrderStatusCode.received, - links=[], - ), - OrderStatus( - timestamp=datetime(2025, 1, 15, 5, 20, 48, 466726, tzinfo=timezone.utc), - status_code=OrderStatusCode.accepted, - links=[], - ), - OrderStatus( - timestamp=datetime(2025, 1, 16, 10, 15, 32, 466726, tzinfo=timezone.utc), - status_code=OrderStatusCode.completed, - links=[], - ), - ] -} diff --git a/tests/conftest.py b/tests/conftest.py index 49f40b0..de73ea1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,6 @@ from stapi_fastapi.routers.root_router import RootRouter from .application import ( - TEST_STATUSES, InMemoryOrderDB, MockProductBackend, MockRootBackend, @@ -42,13 +41,6 @@ def order_db() -> InMemoryOrderDB: return InMemoryOrderDB() -@pytest.fixture -def order_db_statuses() -> InMemoryOrderDB: - order_db = InMemoryOrderDB() - order_db._statuses = TEST_STATUSES - return order_db - - @pytest.fixture def product_backend(order_db: InMemoryOrderDB) -> MockProductBackend: return MockProductBackend(order_db) @@ -59,11 +51,6 @@ def root_backend(order_db: InMemoryOrderDB) -> MockRootBackend: return MockRootBackend(order_db) -@pytest.fixture -def root_backend_preloaded(order_db_statuses: InMemoryOrderDB) -> MockRootBackend: - return MockRootBackend(order_db_statuses) - - @pytest.fixture def mock_product_test_spotlight( product_backend: MockProductBackend, mock_provider: Provider @@ -131,16 +118,6 @@ def empty_stapi_client(root_backend, base_url: str) -> Iterator[TestClient]: yield client -@pytest.fixture -def statuses_client(root_backend_preloaded, base_url: str) -> Iterator[TestClient]: - root_router = RootRouter(root_backend_preloaded) - app = FastAPI() - app.include_router(root_router, prefix="") - - with TestClient(app, base_url=f"{base_url}") as client: - yield client - - @pytest.fixture(scope="session") def url_for(base_url: str) -> Iterator[Callable[[str], str]]: def with_trailing_slash(value: str) -> str: @@ -206,7 +183,6 @@ def mock_test_spotlight_opportunities() -> list[Opportunity]: off_nadir={"minimum": 20, "maximum": 22}, vehicle_id=[1], platform="platform_id", - other_thing="abcd1234", ), ), ] diff --git a/tests/test_order.py b/tests/test_order.py index 4b935b5..03352e3 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime, timedelta, timezone import pytest from fastapi import status @@ -7,9 +7,9 @@ from geojson_pydantic.types import Position2D from httpx import Response -from stapi_fastapi.models.order import OrderPayload +from stapi_fastapi.models.order import OrderPayload, OrderStatus, OrderStatusCode -from .application import MyOrderParameters +from .application import InMemoryOrderDB, MyOrderParameters from .backends import MockProductBackend from .shared import find_link @@ -188,13 +188,46 @@ def test_token_not_found(stapi_client: TestClient) -> None: assert res.status_code == status.HTTP_404_NOT_FOUND -def test_order_status_pagination(statuses_client: TestClient, limit: int = 2) -> None: +@pytest.fixture +def order_statuses() -> dict[str, list[OrderStatus]]: + statuses = { + "test_order_id": [ + OrderStatus( + timestamp=datetime(2025, 1, 14, 2, 21, 48, 466726, tzinfo=timezone.utc), + status_code=OrderStatusCode.received, + links=[], + ), + OrderStatus( + timestamp=datetime(2025, 1, 15, 5, 20, 48, 466726, tzinfo=timezone.utc), + status_code=OrderStatusCode.accepted, + links=[], + ), + OrderStatus( + timestamp=datetime( + 2025, 1, 16, 10, 15, 32, 466726, tzinfo=timezone.utc + ), + status_code=OrderStatusCode.completed, + links=[], + ), + ] + } + return statuses + + +def test_order_status_pagination( + stapi_client: TestClient, + order_db: InMemoryOrderDB, + order_statuses: dict[str, list[OrderStatus]], + limit: int = 2, +) -> None: + order_db._statuses = order_statuses + order_id = "test_order_id" - res = statuses_client.get(f"/orders/{order_id}/statuses") + res = stapi_client.get(f"/orders/{order_id}/statuses") assert res.status_code == status.HTTP_200_OK order_id = "test_order_id" - res = statuses_client.get( + res = stapi_client.get( f"/orders/{order_id}/statuses", params={"next": None, "limit": limit} ) body = res.json() @@ -205,7 +238,7 @@ def test_order_status_pagination(statuses_client: TestClient, limit: int = 2) -> next = link["href"] while len(links) > 1: - res = statuses_client.get(next) + res = stapi_client.get(next) assert res.status_code == status.HTTP_200_OK body = res.json() assert body["statuses"] != [] @@ -217,8 +250,13 @@ def test_order_status_pagination(statuses_client: TestClient, limit: int = 2) -> def test_get_order_statuses_bad_token( - statuses_client: TestClient, limit: int = 2 + stapi_client: TestClient, + order_db: InMemoryOrderDB, + order_statuses: dict[str, list[OrderStatus]], + limit: int = 2, ) -> None: + order_db._statuses = order_statuses + order_id = "non_existing_order_id" - res = statuses_client.get(f"/orders/{order_id}/statuses") + res = stapi_client.get(f"/orders/{order_id}/statuses") assert res.status_code == status.HTTP_404_NOT_FOUND From 2759ec261c4da58555042c5a1d0f32c759811911 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Tue, 14 Jan 2025 16:30:23 -0500 Subject: [PATCH 21/35] feat: adding pagination for POST search_opportunities --- src/stapi_fastapi/backends/product_backend.py | 4 +- src/stapi_fastapi/routers/product_router.py | 46 ++++++++++++------ tests/application.py | 20 ++++---- tests/conftest.py | 7 +++ tests/test_opportunity.py | 47 +++++++++++++++++-- 5 files changed, 93 insertions(+), 31 deletions(-) diff --git a/src/stapi_fastapi/backends/product_backend.py b/src/stapi_fastapi/backends/product_backend.py index 0172500..a947b46 100644 --- a/src/stapi_fastapi/backends/product_backend.py +++ b/src/stapi_fastapi/backends/product_backend.py @@ -16,7 +16,9 @@ async def search_opportunities( product_router: ProductRouter, search: OpportunityRequest, request: Request, - ) -> ResultE[list[Opportunity]]: + next: str | None, + limit: int, + ) -> ResultE[tuple[list[Opportunity], str]]: """ Search for ordering opportunities for the given search parameters. diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index 43de3f0..4fd7685 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -165,27 +165,45 @@ def get_product(self, request: Request) -> Product: ) async def search_opportunities( - self, search: OpportunityRequest, request: Request + self, + search: OpportunityRequest, + request: Request, + next: str | None = None, + limit: int = 10, ) -> OpportunityCollection: """ Explore the opportunities available for a particular set of constraints """ - match await self.product.backend.search_opportunities(self, search, request): - case Success(features): - return OpportunityCollection( - features=features, - links=[ - Link( # current bug is missing method set and setting body for + match await self.product.backend.search_opportunities( + self, search, request, next, limit + ): + case Success((features, pagination_token)): + links = [ + Link( # current bug is missing method set and setting body for + href=str( + request.url_for( + f"{self.root_router.name}:{self.product.id}:create-order", + ), + ), + rel="create-order", + type=TYPE_JSON, + method="POST", + ), + ] + if pagination_token: + links.append( + Link( href=str( - request.url_for( - f"{self.root_router.name}:{self.product.id}:create-order", - ), + request.url.include_query_params(next=pagination_token) ), - rel="create-order", + rel="next", type=TYPE_JSON, - ), - ], - ) + method="POST", + body=search, + ) + ) + return OpportunityCollection(features=features, links=links) + return OpportunityCollection(features=features, links=links) case Failure(e) if isinstance(e, ConstraintsException): raise e case Failure(e): diff --git a/tests/application.py b/tests/application.py index de1ef5f..bb460ba 100644 --- a/tests/application.py +++ b/tests/application.py @@ -113,19 +113,15 @@ async def search_opportunities( if limit > 100: limit = 100 if next: - start = self._opportunities.index(next) + start = int(next) end = min(start + limit, len(self._opportunities)) - print(end) - # opportunities = self._opportunities.k - return Success( - ( - [ - o.model_copy(update=search.model_dump()) - for o in self._opportunities - ], - "", - ) - ) + opportunities = [ + o.model_copy(update=search.model_dump()) + for o in self._opportunities[start:end] + ] + if end < len(self._opportunities): + return Success((opportunities, str(end))) + return Success((opportunities, "")) except Exception as e: return Failure(e) diff --git a/tests/conftest.py b/tests/conftest.py index de73ea1..9e0895c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -186,3 +186,10 @@ def mock_test_spotlight_opportunities() -> list[Opportunity]: ), ), ] + + +@pytest.fixture +def mock_test_pagination_opportunities( + mock_test_spotlight_opportunities, +) -> list[Opportunity]: + return [opp for opp in mock_test_spotlight_opportunities for __ in range(0, 3)] diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index 29ba4d1..bb5e0e3 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -2,6 +2,7 @@ from typing import List import pytest +from fastapi import status from fastapi.testclient import TestClient from stapi_fastapi.models.opportunity import Opportunity, OpportunityCollection @@ -27,7 +28,6 @@ def test_search_opportunities_response( start_string = rfc3339_strftime(start, format) end_string = rfc3339_strftime(end, format) - # Prepare the request payload request_payload = { "geometry": { "type": "Point", @@ -43,13 +43,10 @@ def test_search_opportunities_response( }, } - # Construct the endpoint URL using the `product_name` parameter url = f"/products/{product_id}/opportunities" - # Use POST method to send the payload response = stapi_client.post(url, json=request_payload) - # Validate response status and structure assert response.status_code == 200, f"Failed for product: {product_id}" body = response.json() @@ -59,3 +56,45 @@ def test_search_opportunities_response( pytest.fail("response is not an opportunity collection") assert_link(f"POST {url}", body, "create-order", f"/products/{product_id}/orders") + + +@pytest.mark.parametrize("product_id", ["test-spotlight"]) +def test_search_opportunities_pagination( + product_id: str, + stapi_client: TestClient, + product_backend: MockProductBackend, + mock_test_pagination_opportunities: List[Opportunity], +) -> None: + product_backend._opportunities = mock_test_pagination_opportunities + + now = datetime.now(UTC) + start = now + end = start + timedelta(days=5) + format = "%Y-%m-%dT%H:%M:%S.%f%z" + start_string = rfc3339_strftime(start, format) + end_string = rfc3339_strftime(end, format) + + request_payload = { + "geometry": { + "type": "Point", + "coordinates": [0, 0], + }, + "datetime": f"{start_string}/{end_string}", + "filter": { + "op": "and", + "args": [ + {"op": ">", "args": [{"property": "off_nadir"}, 0]}, + {"op": "<", "args": [{"property": "off_nadir"}, 45]}, + ], + }, + } + + res = stapi_client.post( + f"/products/{product_id}/opportunities", + json=request_payload, + params={"next": None, "limit": 2}, + ) + body = res.json() # noqa: F841 + + assert res.status_code == status.HTTP_200_OK + assert 1 == 2 From 0d5d837845d7b515b593431dc6f15eba5fdb081b Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 16 Jan 2025 11:24:15 -0500 Subject: [PATCH 22/35] tests: abstracted pagination testing out into a more generalized pagination_tester function --- tests/conftest.py | 97 +++++++++++++++++++++++---------------- tests/test_opportunity.py | 40 +++++++++++++++- tests/test_order.py | 59 +++++++----------------- 3 files changed, 113 insertions(+), 83 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e0895c..1cd7200 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,12 @@ from collections.abc import Iterator -from datetime import datetime, timedelta, timezone from typing import Any, Callable from urllib.parse import urljoin -from uuid import uuid4 import pytest -from fastapi import FastAPI +from fastapi import FastAPI, status from fastapi.testclient import TestClient -from geojson_pydantic import Point -from geojson_pydantic.types import Position2D +from httpx import Response -from stapi_fastapi.models.opportunity import ( - Opportunity, -) from stapi_fastapi.models.product import ( Product, Provider, @@ -161,35 +155,58 @@ def mock_provider() -> Provider: ) -@pytest.fixture -def mock_test_spotlight_opportunities() -> list[Opportunity]: - """Fixture to create mock data for Opportunities for `test-spotlight-1`.""" - now = datetime.now(timezone.utc) # Use timezone-aware datetime - start = now - end = start + timedelta(days=5) - - # Create a list of mock opportunities for the given product - return [ - Opportunity( - id=str(uuid4()), - type="Feature", - geometry=Point( - type="Point", - coordinates=Position2D(longitude=0.0, latitude=0.0), - ), - properties=MyOpportunityProperties( - product_id="xyz123", - datetime=(start, end), - off_nadir={"minimum": 20, "maximum": 22}, - vehicle_id=[1], - platform="platform_id", - ), - ), - ] - - -@pytest.fixture -def mock_test_pagination_opportunities( - mock_test_spotlight_opportunities, -) -> list[Opportunity]: - return [opp for opp in mock_test_spotlight_opportunities for __ in range(0, 3)] +def pagination_tester( + stapi_client: TestClient, + endpoint: str, + method: str, + limit: int, + target: str, + expected_total_returns: int, + body: dict | None = None, +) -> None: + retrieved = [] + + res = make_request(stapi_client, endpoint, method, body, None, limit) + assert res.status_code == status.HTTP_200_OK + resp_body = res.json() + + assert len(resp_body[target]) <= limit + retrieved.extend(resp_body[target]) + next_url = next((d["href"] for d in resp_body["links"] if d["rel"] == "next"), None) + + while next_url: + res = make_request(stapi_client, next_url, method, body, next_url, limit) + assert res.status_code == status.HTTP_200_OK + assert len(resp_body[target]) <= limit + resp_body = res.json() + retrieved.extend(resp_body[target]) + + if resp_body["links"]: + next_url = next( + (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None + ) + else: + next_url = None + + assert len(retrieved) == expected_total_returns + + +def make_request( + stapi_client: TestClient, + endpoint: str, + method: str, + body: dict | None, + next_token: str | None, + limit: int, +) -> Response: + """request wrapper for pagination tests""" + if next_token and "next=" in next_token: + next_token = next_token.split("next=")[1] + params = {"next": next_token, "limit": limit} + + if method == "GET": + res = stapi_client.get(endpoint, params=params) + if method == "POST": + res = stapi_client.post(endpoint, json=body, params=params) + + return res diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index bb5e0e3..4f96b78 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -1,16 +1,54 @@ -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime, timedelta, timezone from typing import List +from uuid import uuid4 import pytest from fastapi import status from fastapi.testclient import TestClient +from geojson_pydantic import Point +from geojson_pydantic.types import Position2D from stapi_fastapi.models.opportunity import Opportunity, OpportunityCollection +from tests.application import MyOpportunityProperties from .backends import MockProductBackend from .test_datetime_interval import rfc3339_strftime +@pytest.fixture +def mock_test_spotlight_opportunities() -> list[Opportunity]: + """Fixture to create mock data for Opportunities for `test-spotlight-1`.""" + now = datetime.now(timezone.utc) # Use timezone-aware datetime + start = now + end = start + timedelta(days=5) + + # Create a list of mock opportunities for the given product + return [ + Opportunity( + id=str(uuid4()), + type="Feature", + geometry=Point( + type="Point", + coordinates=Position2D(longitude=0.0, latitude=0.0), + ), + properties=MyOpportunityProperties( + product_id="xyz123", + datetime=(start, end), + off_nadir={"minimum": 20, "maximum": 22}, + vehicle_id=[1], + platform="platform_id", + ), + ), + ] + + +@pytest.fixture +def mock_test_pagination_opportunities( + mock_test_spotlight_opportunities, +) -> list[Opportunity]: + return [opp for opp in mock_test_spotlight_opportunities for __ in range(0, 3)] + + @pytest.mark.parametrize("product_id", ["test-spotlight"]) def test_search_opportunities_response( product_id: str, diff --git a/tests/test_order.py b/tests/test_order.py index 03352e3..77a6e31 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -8,6 +8,7 @@ from httpx import Response from stapi_fastapi.models.order import OrderPayload, OrderStatus, OrderStatusCode +from tests.conftest import pagination_tester from .application import InMemoryOrderDB, MyOrderParameters from .backends import MockProductBackend @@ -161,26 +162,15 @@ def setup_orders_pagination(stapi_client: TestClient, create_order_payloads) -> assert res.headers["Content-Type"] == "application/geo+json" -@pytest.mark.parametrize("limit", [2]) -def test_order_pagination( - stapi_client: TestClient, setup_orders_pagination, limit -) -> None: - res = stapi_client.get("/orders", params={"next": None, "limit": limit}) - assert res.status_code == status.HTTP_200_OK - body = res.json() - assert len(body["features"]) == limit - next = body["links"][0]["href"] - - while next: - res = stapi_client.get(next) - assert res.status_code == status.HTTP_200_OK - body = res.json() - assert body["features"] != [] - if body["links"]: - assert len(body["features"]) == limit - next = body["links"][0]["href"] - else: - break +def test_order_pagination(setup_orders_pagination, stapi_client: TestClient) -> None: + pagination_tester( + stapi_client=stapi_client, + endpoint="/orders", + method="GET", + limit=2, + target="features", + expected_total_returns=3, + ) def test_token_not_found(stapi_client: TestClient) -> None: @@ -223,30 +213,15 @@ def test_order_status_pagination( order_db._statuses = order_statuses order_id = "test_order_id" - res = stapi_client.get(f"/orders/{order_id}/statuses") - assert res.status_code == status.HTTP_200_OK - order_id = "test_order_id" - res = stapi_client.get( - f"/orders/{order_id}/statuses", params={"next": None, "limit": limit} + pagination_tester( + stapi_client=stapi_client, + endpoint=f"/orders/{order_id}/statuses", + method="GET", + limit=2, + target="statuses", + expected_total_returns=3, ) - body = res.json() - links = body["links"] - for link in body["links"]: - if ("rel", "next") in link.items(): - assert len(body["statuses"]) == limit - next = link["href"] - - while len(links) > 1: - res = stapi_client.get(next) - assert res.status_code == status.HTTP_200_OK - body = res.json() - assert body["statuses"] != [] - links = body["links"] - for link in body["links"]: - if ("rel", "next") in link.items(): - assert len(body["statuses"]) == limit - next = body["links"][0]["href"] def test_get_order_statuses_bad_token( From fb9710e5a5b67649932349b1bf70bdefbf00f56a Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 16 Jan 2025 12:02:58 -0500 Subject: [PATCH 23/35] tests: extend pagination tester to inlcude rebuilding and passing POST body tests: add pagination tester to search opportunties test --- src/stapi_fastapi/routers/product_router.py | 2 +- tests/conftest.py | 3 ++ tests/test_opportunity.py | 18 ++++++------ tests/test_product.py | 32 +++++++-------------- 4 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index 4fd7685..202ef88 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -179,7 +179,7 @@ async def search_opportunities( ): case Success((features, pagination_token)): links = [ - Link( # current bug is missing method set and setting body for + Link( href=str( request.url_for( f"{self.root_router.name}:{self.product.id}:create-order", diff --git a/tests/conftest.py b/tests/conftest.py index 1cd7200..62007e9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -185,6 +185,9 @@ def pagination_tester( next_url = next( (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None ) + body = next( + (d.et("body") for d in resp_body["links"] if d.get("body")), None + ) else: next_url = None diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index 4f96b78..4808082 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -3,13 +3,13 @@ from uuid import uuid4 import pytest -from fastapi import status from fastapi.testclient import TestClient from geojson_pydantic import Point from geojson_pydantic.types import Position2D from stapi_fastapi.models.opportunity import Opportunity, OpportunityCollection from tests.application import MyOpportunityProperties +from tests.conftest import pagination_tester from .backends import MockProductBackend from .test_datetime_interval import rfc3339_strftime @@ -127,12 +127,12 @@ def test_search_opportunities_pagination( }, } - res = stapi_client.post( - f"/products/{product_id}/opportunities", - json=request_payload, - params={"next": None, "limit": 2}, + pagination_tester( + stapi_client=stapi_client, + endpoint=f"/products/{product_id}/opportunities", + method="POST", + limit=2, + target="features", + expected_total_returns=3, + body=request_payload, ) - body = res.json() # noqa: F841 - - assert res.status_code == status.HTTP_200_OK - assert 1 == 2 diff --git a/tests/test_product.py b/tests/test_product.py index 054c911..7e50252 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -2,6 +2,8 @@ from fastapi import status from fastapi.testclient import TestClient +from tests.conftest import pagination_tester + def test_products_response(stapi_client: TestClient): res = stapi_client.get("/products") @@ -65,27 +67,15 @@ def test_product_order_parameters_response( assert "s3_path" in json_schema["properties"] -@pytest.mark.parametrize("limit", [1]) -def test_product_pagination(stapi_client: TestClient, limit: int): - res = stapi_client.get("/products", params={"next": None, "limit": limit}) - assert res.status_code == status.HTTP_200_OK - body = res.json() - assert len(body["products"]) == limit - links = body["links"] - for d in body["links"]: - if ("rel", "next") in d.items(): - next = d["href"] - - while len(links) > 1: - res = stapi_client.get(next) - assert res.status_code == status.HTTP_200_OK - body = res.json() - assert body["products"] != [] - links = body["links"] - for d in body["links"]: - if ("rel", "next") in d.items(): - assert len(body["products"]) == limit - next = body["links"][0]["href"] +def test_product_pagination(stapi_client: TestClient): + pagination_tester( + stapi_client=stapi_client, + endpoint="/products", + method="GET", + limit=1, + target="products", + expected_total_returns=2, + ) def test_token_not_found(stapi_client: TestClient) -> None: From adf1f5f1872bd33da401d8b0d0d8ae5b46d31028 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 16 Jan 2025 16:07:41 -0500 Subject: [PATCH 24/35] tests: updating pagination tests and pagination_tester to check exact results to make sure we get back the expected results from pagination, not just checking the number of results --- tests/conftest.py | 10 +++++++--- tests/test_opportunity.py | 5 ++++- tests/test_order.py | 30 ++++++++++++++++++++++++------ tests/test_product.py | 39 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 62007e9..9f32e9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -73,7 +73,7 @@ def mock_product_test_wolf_cola( return Product( id="test-wolf-cola", title="Test Wolf Cola Product", - description="Test product for Wolf Cola for testing GET /product pagination", + description="The right cola for closure", license="CC-BY-4.0", keywords=["test", "satellite", "wolf-cola"], providers=[mock_provider], @@ -161,7 +161,7 @@ def pagination_tester( method: str, limit: int, target: str, - expected_total_returns: int, + expected_returns: list, body: dict | None = None, ) -> None: retrieved = [] @@ -181,6 +181,7 @@ def pagination_tester( resp_body = res.json() retrieved.extend(resp_body[target]) + # get url w/ query params for next call if exists, and POST body if necessary if resp_body["links"]: next_url = next( (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None @@ -191,7 +192,8 @@ def pagination_tester( else: next_url = None - assert len(retrieved) == expected_total_returns + assert len(retrieved) == len(expected_returns) + assert retrieved == expected_returns def make_request( @@ -203,6 +205,8 @@ def make_request( limit: int, ) -> Response: """request wrapper for pagination tests""" + + # extract pagination token if next_token and "next=" in next_token: next_token = next_token.split("next=")[1] params = {"next": next_token, "limit": limit} diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index 4808082..c748a6f 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -104,6 +104,9 @@ def test_search_opportunities_pagination( mock_test_pagination_opportunities: List[Opportunity], ) -> None: product_backend._opportunities = mock_test_pagination_opportunities + expected_returns = [ + x.model_dump(mode="json") for x in mock_test_pagination_opportunities + ] now = datetime.now(UTC) start = now @@ -133,6 +136,6 @@ def test_search_opportunities_pagination( method="POST", limit=2, target="features", - expected_total_returns=3, + expected_returns=expected_returns, body=request_payload, ) diff --git a/tests/test_order.py b/tests/test_order.py index 77a6e31..28c4eac 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -1,3 +1,4 @@ +import copy from datetime import UTC, datetime, timedelta, timezone import pytest @@ -7,7 +8,7 @@ from geojson_pydantic.types import Position2D from httpx import Response -from stapi_fastapi.models.order import OrderPayload, OrderStatus, OrderStatusCode +from stapi_fastapi.models.order import Order, OrderPayload, OrderStatus, OrderStatusCode from tests.conftest import pagination_tester from .application import InMemoryOrderDB, MyOrderParameters @@ -149,27 +150,43 @@ def test_order_status_after_create( @pytest.fixture -def setup_orders_pagination(stapi_client: TestClient, create_order_payloads) -> None: +def setup_orders_pagination( + stapi_client: TestClient, create_order_payloads +) -> list[Order]: product_id = "test-spotlight" - + orders = [] + # t = {'id': 'dc2d9027-a670-475b-878c-a5a6e8ab022b', 'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [14.4, 56.5]}, 'properties': {'product_id': 'test-spotlight', 'created': '2025-01-16T19:09:45.448059Z', 'status': {'timestamp': '2025-01-16T19:09:45.447952Z', 'status_code': 'received', 'reason_code': None, 'reason_text': None, 'links': []}, 'search_parameters': {'datetime': '2024-10-09T18:55:33+00:00/2024-10-12T18:55:33+00:00', 'geometry': {'type': 'Point', 'coordinates': [14.4, 56.5]}, 'filter': None}, 'opportunity_properties': {'datetime': '2024-01-29T12:00:00Z/2024-01-30T12:00:00Z', 'off_nadir': 10}, 'order_parameters': {'s3_path': 's3://my-bucket'}}, 'links': [{'href': 'http://stapiserver/orders/dc2d9027-a670-475b-878c-a5a6e8ab022b', 'rel': 'self', 'type': 'application/geo+json'}, {'href': 'http://stapiserver/orders/dc2d9027-a670-475b-878c-a5a6e8ab022b/statuses', 'rel': 'monitor', 'type': 'application/json'}, {'href': 'http://stapiserver/orders/dc2d9027-a670-475b-878c-a5a6e8ab022b', 'rel': 'self', 'type': 'application/json'}]} for order in create_order_payloads: res = stapi_client.post( f"products/{product_id}/orders", json=order.model_dump(), ) + body = res.json() + orders.append(body) assert res.status_code == status.HTTP_201_CREATED, res.text assert res.headers["Content-Type"] == "application/geo+json" + return orders + + +def test_order_pagination( + setup_orders_pagination, create_order_payloads, stapi_client: TestClient +) -> None: + expected_returns = [] + for order in setup_orders_pagination: + json_link = copy.deepcopy(order["links"][0]) + json_link["type"] = "application/json" + order["links"].append(json_link) + expected_returns.append(order) -def test_order_pagination(setup_orders_pagination, stapi_client: TestClient) -> None: pagination_tester( stapi_client=stapi_client, endpoint="/orders", method="GET", limit=2, target="features", - expected_total_returns=3, + expected_returns=expected_returns, ) @@ -213,6 +230,7 @@ def test_order_status_pagination( order_db._statuses = order_statuses order_id = "test_order_id" + expected_returns = [x.model_dump(mode="json") for x in order_statuses[order_id]] pagination_tester( stapi_client=stapi_client, @@ -220,7 +238,7 @@ def test_order_status_pagination( method="GET", limit=2, target="statuses", - expected_total_returns=3, + expected_returns=expected_returns, ) diff --git a/tests/test_product.py b/tests/test_product.py index 7e50252..8601386 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -67,14 +67,49 @@ def test_product_order_parameters_response( assert "s3_path" in json_schema["properties"] -def test_product_pagination(stapi_client: TestClient): +def test_product_pagination( + stapi_client: TestClient, mock_product_test_spotlight, mock_product_test_wolf_cola +): + expected_returns = [] + for product in [mock_product_test_spotlight, mock_product_test_wolf_cola]: + prod = product.model_dump(mode="json", by_alias=True) + product_id = prod["id"] + prod["links"] = [ + { + "href": f"http://stapiserver/products/{product_id}", + "rel": "self", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/constraints", + "rel": "constraints", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/order-parameters", + "rel": "order-parameters", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/opportunities", + "rel": "opportunities", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/orders", + "rel": "create-order", + "type": "application/json", + }, + ] + expected_returns.append(prod) + pagination_tester( stapi_client=stapi_client, endpoint="/products", method="GET", limit=1, target="products", - expected_total_returns=2, + expected_returns=expected_returns, ) From cc8fd6218ab40b16cebc85df5b153d4f34b27fcd Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 16 Jan 2025 16:49:46 -0500 Subject: [PATCH 25/35] feat: ensure that paginated endpoints can handle limit=0 and return empty set tests: update tests to ensure limit=0 is properly handled --- src/stapi_fastapi/routers/root_router.py | 2 +- tests/application.py | 6 +- tests/test_opportunity.py | 15 +++-- tests/test_order.py | 25 +++++---- tests/test_product.py | 71 +++++++++++++----------- 5 files changed, 66 insertions(+), 53 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index ce6dce6..74568e1 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -160,7 +160,7 @@ def get_products( type=TYPE_JSON, ), ] - if end < len(product_ids): + if end < len(product_ids) and end != 0: links.append( Link( href=str( diff --git a/tests/application.py b/tests/application.py index bb460ba..d80da1b 100644 --- a/tests/application.py +++ b/tests/application.py @@ -60,7 +60,7 @@ async def get_orders( ids = order_ids[start:end] orders = [self._orders_db._orders[order_id] for order_id in ids] - if end < len(order_ids): + if end < len(order_ids) and end != 0: return Success((orders, self._orders_db._orders[order_ids[end]].id)) return Success((orders, "")) except Exception as e: @@ -87,7 +87,7 @@ async def get_order_statuses( end = min(start + limit, len(statuses)) stati = statuses[start:end] - if end < len(statuses): + if end < len(statuses) and end != 0: return Success((stati, str(end))) return Success((stati, "")) except Exception as e: @@ -119,7 +119,7 @@ async def search_opportunities( o.model_copy(update=search.model_dump()) for o in self._opportunities[start:end] ] - if end < len(self._opportunities): + if end < len(self._opportunities) and end != 0: return Success((opportunities, str(end))) return Success((opportunities, "")) except Exception as e: diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index c748a6f..b6a1ef9 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -96,17 +96,20 @@ def test_search_opportunities_response( assert_link(f"POST {url}", body, "create-order", f"/products/{product_id}/orders") -@pytest.mark.parametrize("product_id", ["test-spotlight"]) +@pytest.mark.parametrize("limit", [0, 2, 4]) def test_search_opportunities_pagination( - product_id: str, + limit: int, stapi_client: TestClient, product_backend: MockProductBackend, mock_test_pagination_opportunities: List[Opportunity], ) -> None: + product_id = "test-spotlight" product_backend._opportunities = mock_test_pagination_opportunities - expected_returns = [ - x.model_dump(mode="json") for x in mock_test_pagination_opportunities - ] + expected_returns = [] + if limit != 0: + expected_returns = [ + x.model_dump(mode="json") for x in mock_test_pagination_opportunities + ] now = datetime.now(UTC) start = now @@ -134,7 +137,7 @@ def test_search_opportunities_pagination( stapi_client=stapi_client, endpoint=f"/products/{product_id}/opportunities", method="POST", - limit=2, + limit=limit, target="features", expected_returns=expected_returns, body=request_payload, diff --git a/tests/test_order.py b/tests/test_order.py index 28c4eac..b6cc3e6 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -170,21 +170,23 @@ def setup_orders_pagination( return orders +@pytest.mark.parametrize("limit", [0, 2, 4]) def test_order_pagination( - setup_orders_pagination, create_order_payloads, stapi_client: TestClient + limit, setup_orders_pagination, create_order_payloads, stapi_client: TestClient ) -> None: expected_returns = [] - for order in setup_orders_pagination: - json_link = copy.deepcopy(order["links"][0]) - json_link["type"] = "application/json" - order["links"].append(json_link) - expected_returns.append(order) + if limit != 0: + for order in setup_orders_pagination: + json_link = copy.deepcopy(order["links"][0]) + json_link["type"] = "application/json" + order["links"].append(json_link) + expected_returns.append(order) pagination_tester( stapi_client=stapi_client, endpoint="/orders", method="GET", - limit=2, + limit=limit, target="features", expected_returns=expected_returns, ) @@ -221,22 +223,25 @@ def order_statuses() -> dict[str, list[OrderStatus]]: return statuses +@pytest.mark.parametrize("limit", [0, 2, 4]) def test_order_status_pagination( + limit: int, stapi_client: TestClient, order_db: InMemoryOrderDB, order_statuses: dict[str, list[OrderStatus]], - limit: int = 2, ) -> None: order_db._statuses = order_statuses order_id = "test_order_id" - expected_returns = [x.model_dump(mode="json") for x in order_statuses[order_id]] + expected_returns = [] + if limit != 0: + expected_returns = [x.model_dump(mode="json") for x in order_statuses[order_id]] pagination_tester( stapi_client=stapi_client, endpoint=f"/orders/{order_id}/statuses", method="GET", - limit=2, + limit=limit, target="statuses", expected_returns=expected_returns, ) diff --git a/tests/test_product.py b/tests/test_product.py index 8601386..6c4c595 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -67,47 +67,52 @@ def test_product_order_parameters_response( assert "s3_path" in json_schema["properties"] +@pytest.mark.parametrize("limit", [0, 1, 2, 4]) def test_product_pagination( - stapi_client: TestClient, mock_product_test_spotlight, mock_product_test_wolf_cola + limit: int, + stapi_client: TestClient, + mock_product_test_spotlight, + mock_product_test_wolf_cola, ): expected_returns = [] - for product in [mock_product_test_spotlight, mock_product_test_wolf_cola]: - prod = product.model_dump(mode="json", by_alias=True) - product_id = prod["id"] - prod["links"] = [ - { - "href": f"http://stapiserver/products/{product_id}", - "rel": "self", - "type": "application/json", - }, - { - "href": f"http://stapiserver/products/{product_id}/constraints", - "rel": "constraints", - "type": "application/json", - }, - { - "href": f"http://stapiserver/products/{product_id}/order-parameters", - "rel": "order-parameters", - "type": "application/json", - }, - { - "href": f"http://stapiserver/products/{product_id}/opportunities", - "rel": "opportunities", - "type": "application/json", - }, - { - "href": f"http://stapiserver/products/{product_id}/orders", - "rel": "create-order", - "type": "application/json", - }, - ] - expected_returns.append(prod) + if limit != 0: + for product in [mock_product_test_spotlight, mock_product_test_wolf_cola]: + prod = product.model_dump(mode="json", by_alias=True) + product_id = prod["id"] + prod["links"] = [ + { + "href": f"http://stapiserver/products/{product_id}", + "rel": "self", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/constraints", + "rel": "constraints", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/order-parameters", + "rel": "order-parameters", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/opportunities", + "rel": "opportunities", + "type": "application/json", + }, + { + "href": f"http://stapiserver/products/{product_id}/orders", + "rel": "create-order", + "type": "application/json", + }, + ] + expected_returns.append(prod) pagination_tester( stapi_client=stapi_client, endpoint="/products", method="GET", - limit=1, + limit=limit, target="products", expected_returns=expected_returns, ) From a5999bd552c7b1b58708e8cbeea71caf1477ce63 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 16 Jan 2025 21:21:56 -0500 Subject: [PATCH 26/35] fix: removing uneeded comment that was used in development --- src/stapi_fastapi/routers/product_router.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index 202ef88..1cbf349 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -47,10 +47,6 @@ def __init__( tags=["Products"], ) - # Any paginated POST or requires a body needs the same POST body to be sent - # method property on Link object needs to be set to POST - # next link for GET can just include next token dont need to specify method or body - # where there's a POST body, have to include post body body property is request.body() self.add_api_route( path="/opportunities", endpoint=self.search_opportunities, From 9cfd146ec431ffe93c6ce53716be4699aa5947dd Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 23 Jan 2025 21:39:02 -0500 Subject: [PATCH 27/35] feat: creating methods to make Links to improve readability feat: making pagination token return Maybe[str] instead of usng empty strings feat: tweaking logic path in endpoint functions to make a single return point tests: updating tests as necessary to accomodating changes --- src/stapi_fastapi/backends/product_backend.py | 5 +- src/stapi_fastapi/backends/root_backend.py | 4 +- src/stapi_fastapi/routers/product_router.py | 45 +++---- src/stapi_fastapi/routers/root_router.py | 118 +++++++++--------- tests/application.py | 34 ++--- tests/conftest.py | 41 +++--- tests/test_opportunity.py | 2 +- tests/test_order.py | 9 +- tests/test_product.py | 7 +- 9 files changed, 142 insertions(+), 123 deletions(-) diff --git a/src/stapi_fastapi/backends/product_backend.py b/src/stapi_fastapi/backends/product_backend.py index a947b46..b55104e 100644 --- a/src/stapi_fastapi/backends/product_backend.py +++ b/src/stapi_fastapi/backends/product_backend.py @@ -3,6 +3,7 @@ from typing import Protocol from fastapi import Request +from returns.maybe import Maybe from returns.result import ResultE from stapi_fastapi.models.opportunity import Opportunity, OpportunityRequest @@ -18,9 +19,9 @@ async def search_opportunities( request: Request, next: str | None, limit: int, - ) -> ResultE[tuple[list[Opportunity], str]]: + ) -> ResultE[tuple[list[Opportunity], Maybe[str]]]: """ - Search for ordering opportunities for the given search parameters. + Search for ordering opportunities for the given search parameters and return pagination token if applicable. Backends must validate search constraints and return `stapi_fastapi.exceptions.ConstraintsException` if not valid. diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 427cbf0..4ae9213 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -13,7 +13,7 @@ class RootBackend[T: OrderStatus](Protocol): # pragma: nocover async def get_orders( self, request: Request, next: str | None, limit: int - ) -> ResultE[tuple[list[Order], str]]: + ) -> ResultE[tuple[list[Order], Maybe[str]]]: """ Return a list of existing orders and pagination token if applicable No pagination will return empty string for token @@ -35,7 +35,7 @@ async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Orde async def get_order_statuses( self, order_id: str, request: Request, next: str | None, limit: int - ) -> ResultE[tuple[list[T], str]]: + ) -> ResultE[tuple[list[T], Maybe[str]]]: """ Get statuses for order with `order_id`. diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index 1cbf349..b6e8f0f 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, HTTPException, Request, Response, status from geojson_pydantic.geometries import Geometry +from returns.maybe import Some from returns.result import Failure, Success from stapi_fastapi.constants import TYPE_JSON @@ -170,36 +171,25 @@ async def search_opportunities( """ Explore the opportunities available for a particular set of constraints """ + links: list[Link] = [] match await self.product.backend.search_opportunities( self, search, request, next, limit ): - case Success((features, pagination_token)): - links = [ + case Success((features, Some(pagination_token))): + links.append(self.order_link(request, "create-order")) + links.append( Link( href=str( - request.url_for( - f"{self.root_router.name}:{self.product.id}:create-order", - ), + request.url.remove_query_params(keys=["next", "limit"]) ), - rel="create-order", + rel="next", type=TYPE_JSON, method="POST", - ), - ] - if pagination_token: - links.append( - Link( - href=str( - request.url.include_query_params(next=pagination_token) - ), - rel="next", - type=TYPE_JSON, - method="POST", - body=search, - ) + body={"next": pagination_token, "search": search}, ) - return OpportunityCollection(features=features, links=links) - return OpportunityCollection(features=features, links=links) + ) + case Success((features, Nothing)): # noqa: F841 + links.append(self.order_link(request, "create-order")) case Failure(e) if isinstance(e, ConstraintsException): raise e case Failure(e): @@ -213,6 +203,7 @@ async def search_opportunities( ) case x: raise AssertionError(f"Expected code to be unreachable {x}") + return OpportunityCollection(features=features, links=links) def get_product_constraints(self: Self) -> JsonSchemaModel: """ @@ -255,3 +246,15 @@ async def create_order( ) case x: raise AssertionError(f"Expected code to be unreachable {x}") + + def order_link(self, request: Request, suffix: str): + return Link( + href=str( + request.url_for( + f"{self.root_router.name}:{self.product.id}:{suffix}", + ), + ), + rel="create-order", + type=TYPE_JSON, + method="POST", + ) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 74568e1..4666b15 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -42,6 +42,7 @@ def __init__( self.conformances = conformances self.openapi_endpoint_name = openapi_endpoint_name self.docs_endpoint_name = docs_endpoint_name + self.product_ids: list = [] # A dict is used to track the product routers so we can ensure # idempotentcy in case a product is added multiple times, and also to @@ -144,15 +145,16 @@ def get_products( self, request: Request, next: str | None = None, limit: int = 10 ) -> ProductsCollection: start = 0 - product_ids = [*self.product_routers.keys()] try: if next: - start = product_ids.index(next) + start = self.product_ids.index(next) except ValueError: - logging.exception("An error occurred while retrieving orders") - raise NotFoundException(detail="Error finding pagination token") from None - end = min(start + limit, len(product_ids)) - ids = product_ids[start:end] + logging.exception("An error occurred while retrieving products") + raise NotFoundException( + detail="Error finding pagination token for products" + ) from None + end = start + limit + ids = self.product_ids[start:end] links = [ Link( href=str(request.url_for(f"{self.name}:list-products")), @@ -160,13 +162,11 @@ def get_products( type=TYPE_JSON, ), ] - if end < len(product_ids) and end != 0: + if end > 0 and end < len(self.product_ids): links.append( Link( href=str( - request.url.include_query_params( - next=self.product_routers[product_ids[end]].product.id - ), + request.url.include_query_params(next=self.product_ids[end]), ), rel="next", type=TYPE_JSON, @@ -183,36 +183,24 @@ def get_products( async def get_orders( self, request: Request, next: str | None = None, limit: int = 10 ) -> OrderCollection: + # links: list[Link] = [] match await self.backend.get_orders(request, next, limit): - case Success((orders, pagination_token)): + case Success((orders, Some(pagination_token))): for order in orders: - order.links.append( - Link( - href=str( - request.url_for( - f"{self.name}:get-order", order_id=order.id - ) - ), - rel="self", - type=TYPE_JSON, - ) - ) - if pagination_token: - return OrderCollection( - features=orders, - links=[ - Link( - href=str( - request.url.include_query_params( - next=pagination_token - ) - ), - rel="next", - type=TYPE_JSON, - ) - ], + order.links.append(self.order_link(request, "get-order", order)) + links = [ + Link( + href=str( + request.url.include_query_params(next=pagination_token) + ), + rel="next", + type=TYPE_JSON, ) - return OrderCollection(features=orders) + ] + case Success((orders, Nothing)): # noqa: F841 + for order in orders: + order.links.append(self.order_link(request, "get-order", order)) + links = [] case Failure(e): logger.error( "An error occurred while retrieving orders: %s", @@ -227,6 +215,7 @@ async def get_orders( ) case _: raise AssertionError("Expected code to be unreachable") + return OrderCollection(features=orders, links=links) async def get_order(self: Self, order_id: str, request: Request) -> Order: """ @@ -258,34 +247,24 @@ async def get_order_statuses( next: str | None = None, limit: int = 10, ) -> OrderStatuses: + links: list[Link] = [] match await self.backend.get_order_statuses(order_id, request, next, limit): - case Success((statuses, pagination_token)): - links = [ + case Success((statuses, Some(pagination_token))): + links.append( + self.order_statuses_link(request, "list-order-statuses", order_id) + ) + links.append( Link( href=str( - request.url_for( - f"{self.name}:list-order-statuses", - order_id=order_id, - ) + request.url.include_query_params(next=pagination_token) ), - rel="self", + rel="next", type=TYPE_JSON, ) - ] - if pagination_token: - links.append( - Link( - href=str( - request.url.include_query_params(next=pagination_token) - ), - rel="next", - type=TYPE_JSON, - ) - ) - return OrderStatuses(statuses=statuses, links=links) - return OrderStatuses( - statuses=statuses, - links=links, + ) + case Success((statuses, Nothing)): # noqa: F841 + links.append( + self.order_statuses_link(request, "list-order-statuses", order_id) ) case Failure(e): logger.error( @@ -301,12 +280,14 @@ async def get_order_statuses( ) case _: raise AssertionError("Expected code to be unreachable") + return OrderStatuses(statuses=statuses, links=links) def add_product(self: Self, product: Product, *args, **kwargs) -> None: # Give the include a prefix from the product router product_router = ProductRouter(product, self, *args, **kwargs) self.include_router(product_router, prefix=f"/products/{product.id}") self.product_routers[product.id] = product_router + self.product_ids = [*self.product_routers.keys()] def generate_order_href(self: Self, request: Request, order_id: str) -> URL: return request.url_for(f"{self.name}:get-order", order_id=order_id) @@ -331,3 +312,22 @@ def add_order_links(self, order: Order, request: Request): type=TYPE_JSON, ), ) + + def order_link(self, request: Request, link_suffix: str, order: Order): + return Link( + href=str(request.url_for(f"{self.name}:{link_suffix}", order_id=order.id)), + rel="self", + type=TYPE_JSON, + ) + + def order_statuses_link(self, request: Request, link_suffix: str, order_id: str): + return Link( + href=str( + request.url_for( + f"{self.name}:{link_suffix}", + order_id=order_id, + ) + ), + rel="self", + type=TYPE_JSON, + ) diff --git a/tests/application.py b/tests/application.py index d80da1b..080d948 100644 --- a/tests/application.py +++ b/tests/application.py @@ -5,7 +5,7 @@ from fastapi import FastAPI, Request from pydantic import BaseModel, Field, model_validator -from returns.maybe import Maybe +from returns.maybe import Maybe, Nothing, Some from returns.result import Failure, ResultE, Success from stapi_fastapi.backends.product_backend import ProductBackend @@ -44,7 +44,7 @@ def __init__(self, orders: InMemoryOrderDB) -> None: async def get_orders( self, request: Request, next: str | None, limit: int - ) -> ResultE[tuple[list[Order], str]]: + ) -> ResultE[tuple[list[Order], Maybe[str]]]: """ Return orders from backend. Handle pagination/limit if applicable """ @@ -56,13 +56,15 @@ async def get_orders( if next: start = order_ids.index(next) - end = min(start + limit, len(order_ids)) + end = start + limit ids = order_ids[start:end] orders = [self._orders_db._orders[order_id] for order_id in ids] - if end < len(order_ids) and end != 0: - return Success((orders, self._orders_db._orders[order_ids[end]].id)) - return Success((orders, "")) + if end > 0 and end < len(order_ids): + return Success( + (orders, Some(self._orders_db._orders[order_ids[end]].id)) + ) + return Success((orders, Nothing)) except Exception as e: return Failure(e) @@ -75,7 +77,7 @@ async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Orde async def get_order_statuses( self, order_id: str, request: Request, next: str | None, limit: int - ) -> ResultE[tuple[list[OrderStatus], str]]: + ) -> ResultE[tuple[list[OrderStatus], Maybe[str]]]: try: start = 0 if limit > 100: @@ -84,12 +86,12 @@ async def get_order_statuses( if next: start = int(next) - end = min(start + limit, len(statuses)) + end = start + limit stati = statuses[start:end] - if end < len(statuses) and end != 0: - return Success((stati, str(end))) - return Success((stati, "")) + if end > 0 and end < len(statuses): + return Success((stati, Some(str(end)))) + return Success((stati, Nothing)) except Exception as e: return Failure(e) @@ -107,21 +109,21 @@ async def search_opportunities( request: Request, next: str | None, limit: int, - ) -> ResultE[tuple[list[Opportunity], str]]: + ) -> ResultE[tuple[list[Opportunity], Maybe[str]]]: try: start = 0 if limit > 100: limit = 100 if next: start = int(next) - end = min(start + limit, len(self._opportunities)) + end = start + limit opportunities = [ o.model_copy(update=search.model_dump()) for o in self._opportunities[start:end] ] - if end < len(self._opportunities) and end != 0: - return Success((opportunities, str(end))) - return Success((opportunities, "")) + if end > 0 and end < len(self._opportunities): + return Success((opportunities, Some(str(end)))) + return Success((opportunities, Nothing)) except Exception as e: return Failure(e) diff --git a/tests/conftest.py b/tests/conftest.py index 9f32e9d..209b0cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import copy from collections.abc import Iterator from typing import Any, Callable from urllib.parse import urljoin @@ -66,16 +67,16 @@ def mock_product_test_spotlight( @pytest.fixture -def mock_product_test_wolf_cola( +def mock_product_test_satellite_provider( product_backend: MockProductBackend, mock_provider: Provider ) -> Product: - """Fixture for a mock Wolf Cola product.""" + """Fixture for a mock satellite provider product.""" return Product( - id="test-wolf-cola", - title="Test Wolf Cola Product", - description="The right cola for closure", + id="test-satellite-provider", + title="Satellite Product", + description="A product by a satellite provider", license="CC-BY-4.0", - keywords=["test", "satellite", "wolf-cola"], + keywords=["test", "satellite", "provider"], providers=[mock_provider], links=[], constraints=MyProductConstraints, @@ -89,12 +90,12 @@ def mock_product_test_wolf_cola( def stapi_client( root_backend, mock_product_test_spotlight, - mock_product_test_wolf_cola, + mock_product_test_satellite_provider, base_url: str, ) -> Iterator[TestClient]: root_router = RootRouter(root_backend) root_router.add_product(mock_product_test_spotlight) - root_router.add_product(mock_product_test_wolf_cola) + root_router.add_product(mock_product_test_satellite_provider) app = FastAPI() app.include_router(root_router, prefix="") @@ -172,10 +173,18 @@ def pagination_tester( assert len(resp_body[target]) <= limit retrieved.extend(resp_body[target]) - next_url = next((d["href"] for d in resp_body["links"] if d["rel"] == "next"), None) + next_token = next( + (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None + ) + + while next_token: + url = copy.deepcopy(next_token) + if method == "POST": + next_token = next( + (d["body"]["next"] for d in resp_body["links"] if d["rel"] == "next"), + ) - while next_url: - res = make_request(stapi_client, next_url, method, body, next_url, limit) + res = make_request(stapi_client, url, method, body, next_token, limit) assert res.status_code == status.HTTP_200_OK assert len(resp_body[target]) <= limit resp_body = res.json() @@ -183,17 +192,19 @@ def pagination_tester( # get url w/ query params for next call if exists, and POST body if necessary if resp_body["links"]: - next_url = next( + next_token = next( (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None ) body = next( - (d.et("body") for d in resp_body["links"] if d.get("body")), None + (d.get("body")["search"] for d in resp_body["links"] if d.get("body")), + None, ) else: - next_url = None + next_token = None assert len(retrieved) == len(expected_returns) assert retrieved == expected_returns + # assert retrieved[:2] == expected_returns[:2] def make_request( @@ -207,7 +218,7 @@ def make_request( """request wrapper for pagination tests""" # extract pagination token - if next_token and "next=" in next_token: + if next_token and method == "GET": next_token = next_token.split("next=")[1] params = {"next": next_token, "limit": limit} diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index b6a1ef9..914fd37 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -96,7 +96,7 @@ def test_search_opportunities_response( assert_link(f"POST {url}", body, "create-order", f"/products/{product_id}/orders") -@pytest.mark.parametrize("limit", [0, 2, 4]) +@pytest.mark.parametrize("limit", [0, 1, 2, 4]) def test_search_opportunities_pagination( limit: int, stapi_client: TestClient, diff --git a/tests/test_order.py b/tests/test_order.py index b6cc3e6..09990e8 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -155,7 +155,6 @@ def setup_orders_pagination( ) -> list[Order]: product_id = "test-spotlight" orders = [] - # t = {'id': 'dc2d9027-a670-475b-878c-a5a6e8ab022b', 'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [14.4, 56.5]}, 'properties': {'product_id': 'test-spotlight', 'created': '2025-01-16T19:09:45.448059Z', 'status': {'timestamp': '2025-01-16T19:09:45.447952Z', 'status_code': 'received', 'reason_code': None, 'reason_text': None, 'links': []}, 'search_parameters': {'datetime': '2024-10-09T18:55:33+00:00/2024-10-12T18:55:33+00:00', 'geometry': {'type': 'Point', 'coordinates': [14.4, 56.5]}, 'filter': None}, 'opportunity_properties': {'datetime': '2024-01-29T12:00:00Z/2024-01-30T12:00:00Z', 'off_nadir': 10}, 'order_parameters': {'s3_path': 's3://my-bucket'}}, 'links': [{'href': 'http://stapiserver/orders/dc2d9027-a670-475b-878c-a5a6e8ab022b', 'rel': 'self', 'type': 'application/geo+json'}, {'href': 'http://stapiserver/orders/dc2d9027-a670-475b-878c-a5a6e8ab022b/statuses', 'rel': 'monitor', 'type': 'application/json'}, {'href': 'http://stapiserver/orders/dc2d9027-a670-475b-878c-a5a6e8ab022b', 'rel': 'self', 'type': 'application/json'}]} for order in create_order_payloads: res = stapi_client.post( f"products/{product_id}/orders", @@ -170,8 +169,8 @@ def setup_orders_pagination( return orders -@pytest.mark.parametrize("limit", [0, 2, 4]) -def test_order_pagination( +@pytest.mark.parametrize("limit", [0, 1, 2, 4]) +def test_get_orders_pagination( limit, setup_orders_pagination, create_order_payloads, stapi_client: TestClient ) -> None: expected_returns = [] @@ -223,8 +222,8 @@ def order_statuses() -> dict[str, list[OrderStatus]]: return statuses -@pytest.mark.parametrize("limit", [0, 2, 4]) -def test_order_status_pagination( +@pytest.mark.parametrize("limit", [0, 1, 2, 4]) +def test_get_order_status_pagination( limit: int, stapi_client: TestClient, order_db: InMemoryOrderDB, diff --git a/tests/test_product.py b/tests/test_product.py index 6c4c595..9e32d5b 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -72,11 +72,14 @@ def test_product_pagination( limit: int, stapi_client: TestClient, mock_product_test_spotlight, - mock_product_test_wolf_cola, + mock_product_test_satellite_provider, ): expected_returns = [] if limit != 0: - for product in [mock_product_test_spotlight, mock_product_test_wolf_cola]: + for product in [ + mock_product_test_spotlight, + mock_product_test_satellite_provider, + ]: prod = product.model_dump(mode="json", by_alias=True) product_id = prod["id"] prod["links"] = [ From 35c8f5395ef7b712c78ae661807dfb8d6440dc1f Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 23 Jan 2025 21:41:21 -0500 Subject: [PATCH 28/35] calculate limit override more cleanly) --- src/stapi_fastapi/routers/root_router.py | 1 + tests/application.py | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 4666b15..1722340 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -145,6 +145,7 @@ def get_products( self, request: Request, next: str | None = None, limit: int = 10 ) -> ProductsCollection: start = 0 + limit = min(limit, 100) try: if next: start = self.product_ids.index(next) diff --git a/tests/application.py b/tests/application.py index 080d948..fa9294d 100644 --- a/tests/application.py +++ b/tests/application.py @@ -50,8 +50,7 @@ async def get_orders( """ try: start = 0 - if limit > 100: - limit = 100 + limit = min(limit, 100) order_ids = [*self._orders_db._orders.keys()] if next: @@ -80,8 +79,7 @@ async def get_order_statuses( ) -> ResultE[tuple[list[OrderStatus], Maybe[str]]]: try: start = 0 - if limit > 100: - limit = 100 + limit = min(limit, 100) statuses = self._orders_db._statuses[order_id] if next: @@ -112,8 +110,7 @@ async def search_opportunities( ) -> ResultE[tuple[list[Opportunity], Maybe[str]]]: try: start = 0 - if limit > 100: - limit = 100 + limit = min(limit, 100) if next: start = int(next) end = start + limit From c148b7d72c2c77beec2578d36fb9f0c4d09161c9 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 24 Jan 2025 16:25:10 -0500 Subject: [PATCH 29/35] feat: slightly tweaking the body that is returned by the post request where next token and search params are not their own separat key value pairs but in the same body --- src/stapi_fastapi/routers/product_router.py | 4 +++- tests/conftest.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index b6e8f0f..634c46b 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -177,6 +177,8 @@ async def search_opportunities( ): case Success((features, Some(pagination_token))): links.append(self.order_link(request, "create-order")) + body = search.model_dump() + body["next"] = pagination_token links.append( Link( href=str( @@ -185,7 +187,7 @@ async def search_opportunities( rel="next", type=TYPE_JSON, method="POST", - body={"next": pagination_token, "search": search}, + body=body, ) ) case Success((features, Nothing)): # noqa: F841 diff --git a/tests/conftest.py b/tests/conftest.py index 209b0cf..9d97fce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -196,7 +196,7 @@ def pagination_tester( (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None ) body = next( - (d.get("body")["search"] for d in resp_body["links"] if d.get("body")), + (d.get("body") for d in resp_body["links"] if d.get("body")), None, ) else: From 3a2d9c16d1e81405c7e9e224c88fe9cfb5b884a6 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Mon, 27 Jan 2025 16:35:21 -0500 Subject: [PATCH 30/35] feat: small fixes based on PR feedback. Creating more link creation methods to clean up endpoint business logic. tests: small tweaks to test based on PR feedback --- src/stapi_fastapi/backends/root_backend.py | 9 +-- src/stapi_fastapi/routers/product_router.py | 31 ++++---- src/stapi_fastapi/routers/root_router.py | 88 ++++++++------------- tests/conftest.py | 18 ++--- tests/test_opportunity.py | 8 +- tests/test_product.py | 2 +- 6 files changed, 62 insertions(+), 94 deletions(-) diff --git a/src/stapi_fastapi/backends/root_backend.py b/src/stapi_fastapi/backends/root_backend.py index 4ae9213..fb3d0e6 100644 --- a/src/stapi_fastapi/backends/root_backend.py +++ b/src/stapi_fastapi/backends/root_backend.py @@ -15,8 +15,7 @@ async def get_orders( self, request: Request, next: str | None, limit: int ) -> ResultE[tuple[list[Order], Maybe[str]]]: """ - Return a list of existing orders and pagination token if applicable - No pagination will return empty string for token + Return a list of existing orders and pagination token if applicable. """ ... @@ -26,8 +25,8 @@ async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Orde Should return returns.results.Success[Order] if order is found. - Should return returns.results.Failure[returns.maybe.Nothing] if the order is - not found or if access is denied. + Should return returns.results.Failure[returns.maybe.Nothing] if the + order is not found or if access is denied. A Failure[Exception] will result in a 500. """ @@ -37,7 +36,7 @@ async def get_order_statuses( self, order_id: str, request: Request, next: str | None, limit: int ) -> ResultE[tuple[list[T], Maybe[str]]]: """ - Get statuses for order with `order_id`. + Get statuses for order with `order_id` and return pagination token if applicable Should return returns.results.Success[list[OrderStatus]] if order is found. diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index 634c46b..1ca0959 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -176,22 +176,12 @@ async def search_opportunities( self, search, request, next, limit ): case Success((features, Some(pagination_token))): - links.append(self.order_link(request, "create-order")) - body = search.model_dump() + links.append(self.order_link(request)) + body = search.model_dump(mode="json") body["next"] = pagination_token - links.append( - Link( - href=str( - request.url.remove_query_params(keys=["next", "limit"]) - ), - rel="next", - type=TYPE_JSON, - method="POST", - body=body, - ) - ) + links.append(self.pagination_link(request, body)) case Success((features, Nothing)): # noqa: F841 - links.append(self.order_link(request, "create-order")) + links.append(self.order_link(request)) case Failure(e) if isinstance(e, ConstraintsException): raise e case Failure(e): @@ -249,14 +239,23 @@ async def create_order( case x: raise AssertionError(f"Expected code to be unreachable {x}") - def order_link(self, request: Request, suffix: str): + def order_link(self, request: Request): return Link( href=str( request.url_for( - f"{self.root_router.name}:{self.product.id}:{suffix}", + f"{self.root_router.name}:{self.product.id}:create-order", ), ), rel="create-order", type=TYPE_JSON, method="POST", ) + + def pagination_link(self, request: Request, body: dict[str, str | dict]): + return Link( + href=str(request.url.remove_query_params(keys=["next", "limit"])), + rel="next", + type=TYPE_JSON, + method="POST", + body=body, + ) diff --git a/src/stapi_fastapi/routers/root_router.py b/src/stapi_fastapi/routers/root_router.py index 1722340..34577c1 100644 --- a/src/stapi_fastapi/routers/root_router.py +++ b/src/stapi_fastapi/routers/root_router.py @@ -42,7 +42,7 @@ def __init__( self.conformances = conformances self.openapi_endpoint_name = openapi_endpoint_name self.docs_endpoint_name = docs_endpoint_name - self.product_ids: list = [] + self.product_ids: list[str] = [] # A dict is used to track the product routers so we can ensure # idempotentcy in case a product is added multiple times, and also to @@ -164,15 +164,7 @@ def get_products( ), ] if end > 0 and end < len(self.product_ids): - links.append( - Link( - href=str( - request.url.include_query_params(next=self.product_ids[end]), - ), - rel="next", - type=TYPE_JSON, - ) - ) + links.append(self.pagination_link(request, self.product_ids[end])) return ProductsCollection( products=[ self.product_routers[product_id].get_product(request) @@ -184,36 +176,26 @@ def get_products( async def get_orders( self, request: Request, next: str | None = None, limit: int = 10 ) -> OrderCollection: - # links: list[Link] = [] + links: list[Link] = [] match await self.backend.get_orders(request, next, limit): case Success((orders, Some(pagination_token))): for order in orders: - order.links.append(self.order_link(request, "get-order", order)) - links = [ - Link( - href=str( - request.url.include_query_params(next=pagination_token) - ), - rel="next", - type=TYPE_JSON, - ) - ] + order.links.append(self.order_link(request, order)) + links.append(self.pagination_link(request, pagination_token)) case Success((orders, Nothing)): # noqa: F841 for order in orders: - order.links.append(self.order_link(request, "get-order", order)) - links = [] + order.links.append(self.order_link(request, order)) + case Failure(ValueError()): + raise NotFoundException(detail="Error finding pagination token") case Failure(e): logger.error( "An error occurred while retrieving orders: %s", traceback.format_exception(e), ) - if isinstance(e, ValueError): - raise NotFoundException(detail="Error finding pagination token") - else: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error finding Orders", - ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error finding Orders", + ) case _: raise AssertionError("Expected code to be unreachable") return OrderCollection(features=orders, links=links) @@ -251,34 +233,21 @@ async def get_order_statuses( links: list[Link] = [] match await self.backend.get_order_statuses(order_id, request, next, limit): case Success((statuses, Some(pagination_token))): - links.append( - self.order_statuses_link(request, "list-order-statuses", order_id) - ) - links.append( - Link( - href=str( - request.url.include_query_params(next=pagination_token) - ), - rel="next", - type=TYPE_JSON, - ) - ) + links.append(self.order_statuses_link(request, order_id)) + links.append(self.pagination_link(request, pagination_token)) case Success((statuses, Nothing)): # noqa: F841 - links.append( - self.order_statuses_link(request, "list-order-statuses", order_id) - ) + links.append(self.order_statuses_link(request, order_id)) + case Failure(KeyError()): + raise NotFoundException("Error finding pagination token") case Failure(e): logger.error( "An error occurred while retrieving order statuses: %s", traceback.format_exception(e), ) - if isinstance(e, KeyError): - raise NotFoundException(detail="Error finding pagination token") - else: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error finding Order Statuses", - ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error finding Order Statuses", + ) case _: raise AssertionError("Expected code to be unreachable") return OrderStatuses(statuses=statuses, links=links) @@ -314,21 +283,28 @@ def add_order_links(self, order: Order, request: Request): ), ) - def order_link(self, request: Request, link_suffix: str, order: Order): + def order_link(self, request: Request, order: Order): return Link( - href=str(request.url_for(f"{self.name}:{link_suffix}", order_id=order.id)), + href=str(request.url_for(f"{self.name}:get-order", order_id=order.id)), rel="self", type=TYPE_JSON, ) - def order_statuses_link(self, request: Request, link_suffix: str, order_id: str): + def order_statuses_link(self, request: Request, order_id: str): return Link( href=str( request.url_for( - f"{self.name}:{link_suffix}", + f"{self.name}:list-order-statuses", order_id=order_id, ) ), rel="self", type=TYPE_JSON, ) + + def pagination_link(self, request: Request, pagination_token: str): + return Link( + href=str(request.url.include_query_params(next=pagination_token)), + rel="next", + type=TYPE_JSON, + ) diff --git a/tests/conftest.py b/tests/conftest.py index 9d97fce..9417116 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import copy from collections.abc import Iterator from typing import Any, Callable from urllib.parse import urljoin @@ -173,18 +172,16 @@ def pagination_tester( assert len(resp_body[target]) <= limit retrieved.extend(resp_body[target]) - next_token = next( - (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None - ) + next_url = next((d["href"] for d in resp_body["links"] if d["rel"] == "next"), None) - while next_token: - url = copy.deepcopy(next_token) + while next_url: + url = next_url if method == "POST": - next_token = next( + next_url = next( (d["body"]["next"] for d in resp_body["links"] if d["rel"] == "next"), ) - res = make_request(stapi_client, url, method, body, next_token, limit) + res = make_request(stapi_client, url, method, body, next_url, limit) assert res.status_code == status.HTTP_200_OK assert len(resp_body[target]) <= limit resp_body = res.json() @@ -192,7 +189,7 @@ def pagination_tester( # get url w/ query params for next call if exists, and POST body if necessary if resp_body["links"]: - next_token = next( + next_url = next( (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None ) body = next( @@ -200,11 +197,10 @@ def pagination_tester( None, ) else: - next_token = None + next_url = None assert len(retrieved) == len(expected_returns) assert retrieved == expected_returns - # assert retrieved[:2] == expected_returns[:2] def make_request( diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index 914fd37..38794d4 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -18,8 +18,7 @@ @pytest.fixture def mock_test_spotlight_opportunities() -> list[Opportunity]: """Fixture to create mock data for Opportunities for `test-spotlight-1`.""" - now = datetime.now(timezone.utc) # Use timezone-aware datetime - start = now + start = datetime.now(timezone.utc) # Use timezone-aware datetime end = start + timedelta(days=5) # Create a list of mock opportunities for the given product @@ -60,10 +59,9 @@ def test_search_opportunities_response( product_backend._opportunities = mock_test_spotlight_opportunities now = datetime.now(UTC) - start = now - end = start + timedelta(days=5) + end = now + timedelta(days=5) format = "%Y-%m-%dT%H:%M:%S.%f%z" - start_string = rfc3339_strftime(start, format) + start_string = rfc3339_strftime(now, format) end_string = rfc3339_strftime(end, format) request_payload = { diff --git a/tests/test_product.py b/tests/test_product.py index 9e32d5b..cb2a45b 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -68,7 +68,7 @@ def test_product_order_parameters_response( @pytest.mark.parametrize("limit", [0, 1, 2, 4]) -def test_product_pagination( +def test_get_products_pagination( limit: int, stapi_client: TestClient, mock_product_test_spotlight, From 794985754dd5754815e9edbc9c8b267dfb828b1f Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Wed, 29 Jan 2025 12:41:33 -0500 Subject: [PATCH 31/35] feat: changed so /opportunities no longer can accept query params and next/limit must be passed as key/value pairs in the POST body. These are also now returned in the POST body in the 'next' link object returned by /opportunities tests: update tests to reflect changes in endpoint. POST body is now extracted wholesale from returned link object istead of just the params --- src/stapi_fastapi/routers/product_router.py | 15 +++--- tests/conftest.py | 10 ++-- tests/test_opportunity.py | 55 ++++++++++++--------- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index 1ca0959..e3af43a 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -2,9 +2,9 @@ import logging import traceback -from typing import TYPE_CHECKING, Self +from typing import TYPE_CHECKING, Annotated, Self -from fastapi import APIRouter, HTTPException, Request, Response, status +from fastapi import APIRouter, Body, HTTPException, Request, Response, status from geojson_pydantic.geometries import Geometry from returns.maybe import Some from returns.result import Failure, Success @@ -165,8 +165,8 @@ async def search_opportunities( self, search: OpportunityRequest, request: Request, - next: str | None = None, - limit: int = 10, + next: Annotated[str | None, Body()] = None, + limit: Annotated[int, Body()] = 10, ) -> OpportunityCollection: """ Explore the opportunities available for a particular set of constraints @@ -177,8 +177,11 @@ async def search_opportunities( ): case Success((features, Some(pagination_token))): links.append(self.order_link(request)) - body = search.model_dump(mode="json") - body["next"] = pagination_token + body = { + "search": search.model_dump(mode="json"), + "next": pagination_token, + "limit": limit, + } links.append(self.pagination_link(request, body)) case Success((features, Nothing)): # noqa: F841 links.append(self.order_link(request)) diff --git a/tests/conftest.py b/tests/conftest.py index 9417116..3cc874b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -177,8 +177,8 @@ def pagination_tester( while next_url: url = next_url if method == "POST": - next_url = next( - (d["body"]["next"] for d in resp_body["links"] if d["rel"] == "next"), + body = next( + (d["body"] for d in resp_body["links"] if d["rel"] == "next"), None ) res = make_request(stapi_client, url, method, body, next_url, limit) @@ -192,10 +192,6 @@ def pagination_tester( next_url = next( (d["href"] for d in resp_body["links"] if d["rel"] == "next"), None ) - body = next( - (d.get("body") for d in resp_body["links"] if d.get("body")), - None, - ) else: next_url = None @@ -221,6 +217,6 @@ def make_request( if method == "GET": res = stapi_client.get(endpoint, params=params) if method == "POST": - res = stapi_client.post(endpoint, json=body, params=params) + res = stapi_client.post(endpoint, json=body) return res diff --git a/tests/test_opportunity.py b/tests/test_opportunity.py index 38794d4..56fadef 100644 --- a/tests/test_opportunity.py +++ b/tests/test_opportunity.py @@ -7,7 +7,10 @@ from geojson_pydantic import Point from geojson_pydantic.types import Position2D -from stapi_fastapi.models.opportunity import Opportunity, OpportunityCollection +from stapi_fastapi.models.opportunity import ( + Opportunity, + OpportunityCollection, +) from tests.application import MyOpportunityProperties from tests.conftest import pagination_tester @@ -65,18 +68,21 @@ def test_search_opportunities_response( end_string = rfc3339_strftime(end, format) request_payload = { - "geometry": { - "type": "Point", - "coordinates": [0, 0], - }, - "datetime": f"{start_string}/{end_string}", - "filter": { - "op": "and", - "args": [ - {"op": ">", "args": [{"property": "off_nadir"}, 0]}, - {"op": "<", "args": [{"property": "off_nadir"}, 45]}, - ], + "search": { + "geometry": { + "type": "Point", + "coordinates": [0, 0], + }, + "datetime": f"{start_string}/{end_string}", + "filter": { + "op": "and", + "args": [ + {"op": ">", "args": [{"property": "off_nadir"}, 0]}, + {"op": "<", "args": [{"property": "off_nadir"}, 45]}, + ], + }, }, + "limit": 10, } url = f"/products/{product_id}/opportunities" @@ -117,18 +123,21 @@ def test_search_opportunities_pagination( end_string = rfc3339_strftime(end, format) request_payload = { - "geometry": { - "type": "Point", - "coordinates": [0, 0], - }, - "datetime": f"{start_string}/{end_string}", - "filter": { - "op": "and", - "args": [ - {"op": ">", "args": [{"property": "off_nadir"}, 0]}, - {"op": "<", "args": [{"property": "off_nadir"}, 45]}, - ], + "search": { + "geometry": { + "type": "Point", + "coordinates": [0, 0], + }, + "datetime": f"{start_string}/{end_string}", + "filter": { + "op": "and", + "args": [ + {"op": ">", "args": [{"property": "off_nadir"}, 0]}, + {"op": "<", "args": [{"property": "off_nadir"}, 45]}, + ], + }, }, + "limit": limit, } pagination_tester( From a19bd3409a3e56bb074d94638a918c3189893570 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Wed, 29 Jan 2025 15:29:50 -0500 Subject: [PATCH 32/35] docs: updating README.md with pagination information --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index e445061..38c278a 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,18 @@ guaranteed to be not correct. STAPI FastAPI provides an `fastapi.APIRouter` which must be included in `fastapi.FastAPI` instance. +### Pagination + +4 endpoints currently offer pagination: +`GET`: `'/orders`, `/products`, `/orders/{order_id}/statuses` +`POST`: `/opportunities`. + +Pagination is token based and follows recommendations in the [STAC API pagination]. Limit and token are passed in as query params for `GET` endpoints, and via the body aas separte key/value pairs for `POST` requests. + +If pagination is available and more records remain the response object will contain a `next` link object that can be used to get the next page of results. No `next` `Link` returned indicates there are no further records available. + +`limit` defaults to 10 and maxes at 100. + ## ADRs @@ -59,3 +71,4 @@ With the `uvicorn` defaults the app should be accessible at [STAPI spec]: https://github.com/stapi-spec/stapi-spec [poetry]: https://python-poetry.org/ +[STAC API pagination]: https://github.com/radiantearth/stac-api-spec/blob/release/v1.0.0/item-search/examples.md#paging-examples From 4d17e1c78d977536bd022ba48f5c01395927a643 Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Wed, 29 Jan 2025 15:43:14 -0500 Subject: [PATCH 33/35] set python version to 3.12 --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 15e0afb..df47d12 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.x" + python-version: "3.12.x" - name: Install dependencies run: | python -m pip install --upgrade pip From 3ef8146e44f9db1ad7a030ec8d3a199406f53a7d Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Thu, 30 Jan 2025 14:43:42 -0500 Subject: [PATCH 34/35] tests: small fixes based on PR comments --- tests/conftest.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3cc874b..cb22fd9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -209,14 +209,13 @@ def make_request( ) -> Response: """request wrapper for pagination tests""" - # extract pagination token - if next_token and method == "GET": - next_token = next_token.split("next=")[1] - params = {"next": next_token, "limit": limit} - - if method == "GET": - res = stapi_client.get(endpoint, params=params) - if method == "POST": - res = stapi_client.post(endpoint, json=body) + match method: + case "GET": + if next_token: # extract pagination token + next_token = next_token.split("next=")[1] + params = {"next": next_token, "limit": limit} + res = stapi_client.get(endpoint, params=params) + case "POST": + res = stapi_client.post(endpoint, json=body) return res From 33a9d3855019ba89dcf99499540c37099140e3bd Mon Sep 17 00:00:00 2001 From: Theodore Reuter Date: Fri, 31 Jan 2025 13:00:58 -0500 Subject: [PATCH 35/35] test: add fail case to make_request match/case --- tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index cb22fd9..cc2261c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ from fastapi import FastAPI, status from fastapi.testclient import TestClient from httpx import Response +from pytest import fail from stapi_fastapi.models.product import ( Product, @@ -217,5 +218,7 @@ def make_request( res = stapi_client.get(endpoint, params=params) case "POST": res = stapi_client.post(endpoint, json=body) + case _: + fail(f"method {method} not supported in make request") return res