From 123805f8c9469891abf8443c4ecab6c736e8d7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Wed, 11 Feb 2026 00:38:22 +0300 Subject: [PATCH 01/17] Try ti reduce saga storage overhead --- README.md | 177 +++++++-------- examples/saga.py | 11 +- examples/saga_fallback.py | 5 +- examples/saga_fastapi_sse.py | 4 +- examples/saga_recovery.py | 21 +- examples/saga_recovery_scheduler.py | 4 +- examples/saga_sqlalchemy_storage.py | 10 +- src/cqrs/saga/compensation.py | 20 +- src/cqrs/saga/execution.py | 12 +- src/cqrs/saga/saga.py | 161 +++++++------ src/cqrs/saga/storage/__init__.py | 3 +- src/cqrs/saga/storage/memory.py | 87 +++++++- src/cqrs/saga/storage/protocol.py | 67 ++++++ src/cqrs/saga/storage/sqlalchemy.py | 165 +++++++++++++- .../dataclasses/test_benchmark_saga_memory.py | 113 +++++++++- .../test_benchmark_saga_sqlalchemy.py | 95 +++++++- .../default/test_benchmark_saga_memory.py | 113 +++++++++- .../default/test_benchmark_saga_sqlalchemy.py | 95 +++++++- tests/unit/test_saga/test_saga_storage_run.py | 211 ++++++++++++++++++ 19 files changed, 1136 insertions(+), 238 deletions(-) create mode 100644 tests/unit/test_saga/test_saga_storage_run.py diff --git a/README.md b/README.md index 4db91ad..5b09fe0 100644 --- a/README.md +++ b/README.md @@ -43,29 +43,29 @@ ## Overview -This is a package for implementing the CQRS (Command Query Responsibility Segregation) pattern in Python applications. -It provides a set of abstractions and utilities to help separate read and write use cases, ensuring better scalability, -performance, and maintainability of the application. +An event-driven framework for building distributed systems in Python. It centers on CQRS (Command Query Responsibility Segregation) and extends into messaging, sagas, and reliable event delivery — so you can separate read and write flows, react to events from the bus, run distributed transactions with compensation, and publish events via Transaction Outbox. The result is clearer structure, better scalability, and easier evolution of the application. This package is a fork of the [diator](https://github.com/akhundMurad/diator) -project ([documentation](https://akhundmurad.github.io/diator/)) with several enhancements: - -1. Support for Pydantic [v2.*](https://docs.pydantic.dev/2.8/); -2. `Kafka` support using [aiokafka](https://github.com/aio-libs/aiokafka); -3. Added `EventMediator` for handling `Notification` and `ECST` events coming from the bus; -4. Redesigned the event and request mapping mechanism to handlers; -5. Added `bootstrap` for easy setup; -6. Added support for [Transaction Outbox](https://microservices.io/patterns/data/transactional-outbox.html), ensuring - that `Notification` and `ECST` events are sent to the broker; -7. FastAPI supporting; -8. FastStream supporting; -9. [Protobuf](https://protobuf.dev/) events supporting; -10. `StreamingRequestMediator` and `StreamingRequestHandler` for handling streaming requests with real-time progress updates; -11. Parallel event processing with configurable concurrency limits; -12. Chain of Responsibility pattern support with `CORRequestHandler` for processing requests through multiple handlers in sequence; -13. Orchestrated Saga pattern support for managing distributed transactions with automatic compensation and recovery mechanisms; -14. Built-in Mermaid diagram generation, enabling automatic generation of Sequence and Class diagrams for documentation and visualization; -15. Flexible Request and Response types support - use Pydantic-based or Dataclass-based implementations, with the ability to mix and match types based on your needs. +project ([documentation](https://akhundmurad.github.io/diator/)) with several enhancements, ordered by importance: + +**Core framework** + +1. Redesigned the event and request mapping mechanism to handlers; +2. `EventMediator` for handling `Notification` and `ECST` events coming from the bus; +3. `bootstrap` for easy setup; +4. **Transaction Outbox**, ensuring that `Notification` and `ECST` events are sent to the broker; +5. **Orchestrated Saga** pattern for distributed transactions with automatic compensation and recovery; +6. `StreamingRequestMediator` and `StreamingRequestHandler` for streaming requests with real-time progress updates; +7. **Chain of Responsibility** with `CORRequestHandler` for processing requests through multiple handlers in sequence; +8. **Parallel event processing** with configurable concurrency limits. + +**Also** + +- **Typing:** Pydantic [v2.*](https://docs.pydantic.dev/2.8/) and `IRequest`/`IResponse` interfaces — use Pydantic-based, dataclass-based, or custom Request/Response implementations. +- **Broker:** Kafka via [aiokafka](https://github.com/aio-libs/aiokafka). +- **Integration:** Ready for integration with FastAPI and FastStream. +- **Documentation:** Built-in Mermaid diagram generation (Sequence and Class diagrams). +- **Protobuf:** Interface-level support for converting Notification events to Protobuf and back. ## Request Handlers @@ -304,6 +304,63 @@ class CustomResponse(cqrs.IResponse): A complete example can be found in [request_response_types.py](https://github.com/vadikko2/cqrs/blob/master/examples/request_response_types.py) +## Mapping + +To bind commands, queries and events with specific handlers, you can use the registries `EventMap` and `RequestMap`. + +```python +from cqrs import requests, events + +from app import commands, command_handlers +from app import queries, query_handlers +from app import events as event_models, event_handlers + + +def init_commands(mapper: requests.RequestMap) -> None: + mapper.bind(commands.JoinMeetingCommand, command_handlers.JoinMeetingCommandHandler) + +def init_queries(mapper: requests.RequestMap) -> None: + mapper.bind(queries.ReadMeetingQuery, query_handlers.ReadMeetingQueryHandler) + +def init_events(mapper: events.EventMap) -> None: + mapper.bind(events.NotificationEvent[events_models.NotificationMeetingRoomClosed], event_handlers.MeetingRoomClosedNotificationHandler) + mapper.bind(events.NotificationEvent[event_models.ECSTMeetingRoomClosed], event_handlers.UpdateMeetingRoomReadModelHandler) +``` + +## Bootstrap + +The `python-cqrs` package implements a set of bootstrap utilities designed to simplify the initial configuration of an +application. + +```python +import functools + +from cqrs.events import bootstrap as event_bootstrap +from cqrs.requests import bootstrap as request_bootstrap + +from app import dependencies, mapping, orm + + +@functools.lru_cache +def mediator_factory(): + return request_bootstrap.bootstrap( + di_container=dependencies.setup_di(), + commands_mapper=mapping.init_commands, + queries_mapper=mapping.init_queries, + domain_events_mapper=mapping.init_events, + on_startup=[orm.init_store_event_mapper], + ) + + +@functools.lru_cache +def event_mediator_factory(): + return event_bootstrap.bootstrap( + di_container=dependencies.setup_di(), + events_mapper=mapping.init_events, + on_startup=[orm.init_store_event_mapper], + ) +``` + ## Saga Pattern The package implements the Orchestrated Saga pattern for managing distributed transactions across multiple services or operations. @@ -689,17 +746,7 @@ loop.run_until_complete(periodically_task()) A complete example can be found in the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/kafka_outboxed_event_producing.py) -## Transaction log tailing - -If the Outbox polling strategy does not suit your needs, I recommend exploring -the [Transaction Log Tailing](https://microservices.io/patterns/data/transaction-log-tailing.html) pattern. -The current version of the python-cqrs package does not support the implementation of this pattern. - -> [!TIP] -> However, it can be implemented -> using [Debezium + Kafka Connect](https://debezium.io/documentation/reference/stable/architecture.html), -> which allows you to produce all newly created events within the Outbox storage directly to the corresponding topic in -> Kafka (or any other broker). +**Transaction log tailing.** If Outbox polling does not suit you, consider [Transaction Log Tailing](https://microservices.io/patterns/data/transaction-log-tailing.html). The package does not implement it; you can use [Debezium + Kafka Connect](https://debezium.io/documentation/reference/stable/architecture.html) to tail the Outbox and produce events to Kafka. ## DI container @@ -765,65 +812,10 @@ Complete examples can be found in: - [Simple example](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injector_integration_simple_example.py) - [Practical example with FastAPI](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injector_integration_practical_example.py) -## Mapping - -To bind commands, queries and events with specific handlers, you can use the registries `EventMap` and `RequestMap`. - -```python -from cqrs import requests, events - -from app import commands, command_handlers -from app import queries, query_handlers -from app import events as event_models, event_handlers - - -def init_commands(mapper: requests.RequestMap) -> None: - mapper.bind(commands.JoinMeetingCommand, command_handlers.JoinMeetingCommandHandler) - -def init_queries(mapper: requests.RequestMap) -> None: - mapper.bind(queries.ReadMeetingQuery, query_handlers.ReadMeetingQueryHandler) - -def init_events(mapper: events.EventMap) -> None: - mapper.bind(events.NotificationEvent[events_models.NotificationMeetingRoomClosed], event_handlers.MeetingRoomClosedNotificationHandler) - mapper.bind(events.NotificationEvent[event_models.ECSTMeetingRoomClosed], event_handlers.UpdateMeetingRoomReadModelHandler) -``` - -## Bootstrap - -The `python-cqrs` package implements a set of bootstrap utilities designed to simplify the initial configuration of an -application. - -```python -import functools - -from cqrs.events import bootstrap as event_bootstrap -from cqrs.requests import bootstrap as request_bootstrap - -from app import dependencies, mapping, orm - - -@functools.lru_cache -def mediator_factory(): - return request_bootstrap.bootstrap( - di_container=dependencies.setup_di(), - commands_mapper=mapping.init_commands, - queries_mapper=mapping.init_queries, - domain_events_mapper=mapping.init_events, - on_startup=[orm.init_store_event_mapper], - ) - - -@functools.lru_cache -def event_mediator_factory(): - return event_bootstrap.bootstrap( - di_container=dependencies.setup_di(), - events_mapper=mapping.init_events, - on_startup=[orm.init_store_event_mapper], - ) -``` - ## Integration with presentation layers +The framework is ready for integration with **FastAPI** and **FastStream**. + > [!TIP] > I recommend reading the useful > paper [Onion Architecture Used in Software Development](https://www.researchgate.net/publication/371006360_Onion_Architecture_Used_in_Software_Development). @@ -956,8 +948,5 @@ the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/fastap ## Protobuf messaging -The `python-cqrs` package supports integration with [protobuf](https://developers.google.com/protocol-buffers/).\\ -Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data – -think XML, but smaller, faster, and simpler. You define how you want your data to be structured once, then you can use -special generated source code to easily write and read your structured data to and from a variety of data streams and -using a variety of languages. +The `python-cqrs` package supports integration with [protobuf](https://developers.google.com/protocol-buffers/). +There is interface-level support for converting Notification events to Protobuf and back. Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data – think XML, but smaller, faster, and simpler. You define how you want your data to be structured once, then you can use special generated source code to easily write and read your structured data to and from a variety of data streams and using a variety of languages. diff --git a/examples/saga.py b/examples/saga.py index 52ae06c..b4956ce 100644 --- a/examples/saga.py +++ b/examples/saga.py @@ -55,6 +55,8 @@ 4. Saga Storage and Logging: - SagaStorage persists saga state and execution history + - MemorySagaStorage and SqlAlchemySagaStorage support create_run(): execution + uses one session per saga and checkpoint commits (fewer commits, better performance) - Each step execution is logged (act/compensate, status, timestamp) - Storage enables recovery of interrupted sagas - Use storage.get_step_history() to view execution log @@ -283,8 +285,7 @@ async def create_shipment( self._shipments[shipment_id] = tracking_number print( - f" ✓ Created shipment {shipment_id} for order {order_id} " - f"(tracking: {tracking_number})", + f" ✓ Created shipment {shipment_id} for order {order_id} " f"(tracking: {tracking_number})", ) return shipment_id, tracking_number @@ -478,8 +479,10 @@ async def run_successful_saga() -> None: payment_service = PaymentService() shipping_service = ShippingService() - # Create saga storage for persistence - # In production, use SQLAlchemySagaStorage or another persistent storage + # Create saga storage for persistence. + # MemorySagaStorage (and SqlAlchemySagaStorage) support create_run(): + # execution uses one session per saga and checkpoint commits for better performance. + # In production, use SqlAlchemySagaStorage or another persistent storage. storage = MemorySagaStorage() # Setup DI container diff --git a/examples/saga_fallback.py b/examples/saga_fallback.py index 5020565..407b162 100644 --- a/examples/saga_fallback.py +++ b/examples/saga_fallback.py @@ -147,8 +147,7 @@ async def act( """Primary step that always raises an error.""" self._call_count += 1 logger.info( - f" [PrimaryStep] Executing act() for order {context.order_id} " - f"(call #{self._call_count})...", + f" [PrimaryStep] Executing act() for order {context.order_id} " f"(call #{self._call_count})...", ) raise RuntimeError("Primary step failed - service unavailable") @@ -302,7 +301,7 @@ async def main() -> None: ), ) - # Create saga storage + # Create saga storage (supports create_run(): one session per saga, checkpoint commits) storage = MemorySagaStorage() di_container.bind( di.bind_by_type( diff --git a/examples/saga_fastapi_sse.py b/examples/saga_fastapi_sse.py index 32c4ef0..8cb8b86 100644 --- a/examples/saga_fastapi_sse.py +++ b/examples/saga_fastapi_sse.py @@ -89,6 +89,8 @@ - Saga state and execution history are persisted to SagaStorage 3. Saga Storage and Logging: + - MemorySagaStorage/SqlAlchemySagaStorage support create_run(): one session per saga, + checkpoint commits (fewer commits, better performance) - SagaStorage persists saga state and execution history - Each step execution is logged (act/compensate, status, timestamp) - Storage enables recovery of interrupted sagas @@ -400,7 +402,7 @@ class OrderSaga(Saga[OrderContext]): # DI Container Setup # ============================================================================ -# Shared storage instance (in production, use persistent storage) +# Shared storage (MemorySagaStorage uses create_run(): scoped run, checkpoint commits) saga_storage = MemorySagaStorage() # Setup DI container diff --git a/examples/saga_recovery.py b/examples/saga_recovery.py index 2bd4e77..72dfd7b 100644 --- a/examples/saga_recovery.py +++ b/examples/saga_recovery.py @@ -57,6 +57,8 @@ ================================================================================ 1. Saga State Persistence: + - MemorySagaStorage/SqlAlchemySagaStorage use create_run(): one session per saga, + checkpoint commits after each step (fewer commits, better performance) - Saga state is saved to storage after each step - Storage tracks which steps completed successfully - Context data is persisted for recovery @@ -289,8 +291,7 @@ async def create_shipment( self._shipments[shipment_id] = tracking_number logger.info( - f" ✓ Created shipment {shipment_id} for order {order_id} " - f"(tracking: {tracking_number})", + f" ✓ Created shipment {shipment_id} for order {order_id} " f"(tracking: {tracking_number})", ) return shipment_id, tracking_number @@ -534,7 +535,7 @@ async def simulate_interrupted_saga() -> tuple[uuid.UUID, MemorySagaStorage]: print("=" * 70) print("\nSimulating server crash after first step...") - # Setup services and storage + # Setup services and storage (MemorySagaStorage uses create_run(): scoped run, checkpoint commits) inventory_service = InventoryService() payment_service = PaymentService() shipping_service = ShippingService() @@ -604,8 +605,7 @@ async def simulate_interrupted_saga() -> tuple[uuid.UUID, MemorySagaStorage]: print("\n Execution log (SagaLog) before recovery:") for entry in history: print( - f" [{entry.timestamp.strftime('%H:%M:%S')}] " - f"{entry.step_name}.{entry.action}: {entry.status.value}", + f" [{entry.timestamp.strftime('%H:%M:%S')}] " f"{entry.step_name}.{entry.action}: {entry.status.value}", ) print("\n⚠️ Problem: Order is incomplete!") @@ -676,8 +676,7 @@ async def recover_interrupted_saga( print("\n Execution log (SagaLog):") for entry in history: print( - f" [{entry.timestamp.strftime('%H:%M:%S')}] " - f"{entry.step_name}.{entry.action}: {entry.status.value}", + f" [{entry.timestamp.strftime('%H:%M:%S')}] " f"{entry.step_name}.{entry.action}: {entry.status.value}", ) print("\n✅ System is now in consistent state!") @@ -699,7 +698,7 @@ async def simulate_interrupted_compensation() -> tuple[uuid.UUID, MemorySagaStor print("=" * 70) print("\nSimulating failure during compensation...") - # Setup services + # Setup services and storage (scoped run via create_run() when supported) inventory_service = InventoryService() payment_service = PaymentService() shipping_service = ShippingService() @@ -786,8 +785,7 @@ async def resolve_with_failing_step(type_: type) -> typing.Any: print("\n Execution log (SagaLog) before recovery:") for entry in history: print( - f" [{entry.timestamp.strftime('%H:%M:%S')}] " - f"{entry.step_name}.{entry.action}: {entry.status.value}", + f" [{entry.timestamp.strftime('%H:%M:%S')}] " f"{entry.step_name}.{entry.action}: {entry.status.value}", ) print("\n⚠️ Problem: Compensation incomplete!") @@ -852,8 +850,7 @@ async def recover_interrupted_compensation( print("\n Execution log (SagaLog):") for entry in history: print( - f" [{entry.timestamp.strftime('%H:%M:%S')}] " - f"{entry.step_name}.{entry.action}: {entry.status.value}", + f" [{entry.timestamp.strftime('%H:%M:%S')}] " f"{entry.step_name}.{entry.action}: {entry.status.value}", ) print("\n✅ System is now in consistent state!") diff --git a/examples/saga_recovery_scheduler.py b/examples/saga_recovery_scheduler.py index 55d2867..a3f3103 100644 --- a/examples/saga_recovery_scheduler.py +++ b/examples/saga_recovery_scheduler.py @@ -50,6 +50,8 @@ - while True with asyncio.sleep(interval_seconds) - get_sagas_for_recovery(limit, max_recovery_attempts, stale_after_seconds) - Per-saga recover_saga() only; increment_recovery_attempts is done inside recover_saga on failure + - When storage supports create_run() (e.g. MemorySagaStorage, SqlAlchemySagaStorage), + saga execution uses one session per saga and checkpoint commits 2. Staleness filter (stale_after_seconds): - Only sagas not updated recently are considered (avoids recovering @@ -631,7 +633,7 @@ async def main() -> None: " 3. recover_saga() per saga (increment_recovery_attempts on failure is internal)", ) - storage = MemorySagaStorage() + storage = MemorySagaStorage() # supports create_run(): scoped run when executing sagas saga_id = await create_interrupted_saga(storage) storage._sagas[saga_id]["updated_at"] = datetime.datetime.now( diff --git a/examples/saga_sqlalchemy_storage.py b/examples/saga_sqlalchemy_storage.py index 5380edb..2daa543 100644 --- a/examples/saga_sqlalchemy_storage.py +++ b/examples/saga_sqlalchemy_storage.py @@ -8,8 +8,9 @@ Key features demonstrated: 1. Configuring SQLAlchemy async engine with connection pooling 2. Initializing SqlAlchemySagaStorage with a session factory -3. Executing sagas with persistent state in a database (SQLite in this example) -4. Handling transaction management automatically via the storage +3. Executing sagas with persistent state in a database (SQLite/MySQL in this example) +4. Scoped run: storage.create_run() is used automatically—one session per saga, + checkpoint commits after each step (fewer commits and sessions than the legacy path) Requirements: pip install sqlalchemy[asyncio] aiosqlite @@ -153,11 +154,12 @@ async def main() -> None: await setup_database(engine) # 3. Create Session Factory - # This factory will be used by the storage to create short-lived sessions for each operation + # Used by the storage; when the saga runs, create_run() yields one session per saga + # with checkpoint commits (after each step), reducing round-trips vs legacy path. session_factory = async_sessionmaker(engine, expire_on_commit=False) # 4. Initialize SqlAlchemySagaStorage - # We pass the session factory, allowing the storage to manage its own transactions + # Supports create_run(): execution uses one session per saga and checkpoint commits. saga_storage = SqlAlchemySagaStorage(session_factory) # 5. Setup Dependency Injection diff --git a/src/cqrs/saga/compensation.py b/src/cqrs/saga/compensation.py index 8c7f194..6b0280a 100644 --- a/src/cqrs/saga/compensation.py +++ b/src/cqrs/saga/compensation.py @@ -7,7 +7,7 @@ from cqrs.saga.models import ContextT from cqrs.saga.step import SagaStepHandler from cqrs.saga.storage.enums import SagaStepStatus, SagaStatus -from cqrs.saga.storage.protocol import ISagaStorage +from cqrs.saga.storage.protocol import ISagaStorage, SagaStorageRun logger = logging.getLogger("cqrs.saga") @@ -19,10 +19,11 @@ def __init__( self, saga_id: typing.Any, context: ContextT, - storage: ISagaStorage, + storage: ISagaStorage | SagaStorageRun, retry_count: int = 3, retry_delay: float = 1.0, retry_backoff: float = 2.0, + on_after_compensate_step: typing.Callable[[], typing.Awaitable[None]] | None = None, ) -> None: """ Initialize compensator. @@ -30,10 +31,12 @@ def __init__( Args: saga_id: UUID of the saga context: Saga context - storage: Saga storage implementation + storage: Saga storage implementation (or run object with same interface) retry_count: Number of retry attempts for compensation retry_delay: Initial delay between retries in seconds retry_backoff: Backoff multiplier for exponential delay + on_after_compensate_step: Optional async callback after each successfully + compensated step (e.g. run.commit() for checkpoint). """ self._saga_id = saga_id self._context = context @@ -41,6 +44,7 @@ def __init__( self._retry_count = retry_count self._retry_delay = retry_delay self._retry_backoff = retry_backoff + self._on_after_compensate_step = on_after_compensate_step async def compensate_steps( self, @@ -65,14 +69,10 @@ async def compensate_steps( # Load history to skip already compensated steps history = await self._storage.get_step_history(self._saga_id) compensated_steps = { - e.step_name - for e in history - if e.status == SagaStepStatus.COMPLETED and e.action == "compensate" + e.step_name for e in history if e.status == SagaStepStatus.COMPLETED and e.action == "compensate" } - compensation_errors: list[ - tuple[SagaStepHandler[ContextT, typing.Any], Exception] - ] = [] + compensation_errors: list[tuple[SagaStepHandler[ContextT, typing.Any], Exception]] = [] for step in reversed(completed_steps): step_name = step.__class__.__name__ @@ -101,6 +101,8 @@ async def compensate_steps( "compensate", SagaStepStatus.COMPLETED, ) + if self._on_after_compensate_step is not None: + await self._on_after_compensate_step() except Exception as compensation_error: await self._storage.log_step( diff --git a/src/cqrs/saga/execution.py b/src/cqrs/saga/execution.py index 4bfa14f..d5ca0f7 100644 --- a/src/cqrs/saga/execution.py +++ b/src/cqrs/saga/execution.py @@ -10,7 +10,7 @@ from cqrs.saga.models import ContextT, SagaContext from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.enums import SagaStepStatus -from cqrs.saga.storage.protocol import ISagaStorage +from cqrs.saga.storage.protocol import ISagaStorage, SagaStorageRun logger = logging.getLogger("cqrs.saga") @@ -21,7 +21,7 @@ class SagaStateManager: def __init__( self, saga_id: typing.Any, - storage: ISagaStorage, + storage: ISagaStorage | SagaStorageRun, ) -> None: """ Initialize state manager. @@ -79,7 +79,7 @@ class SagaRecoveryManager: def __init__( self, saga_id: typing.Any, - storage: ISagaStorage, + storage: ISagaStorage | SagaStorageRun, container: Container, saga_steps: list[type[SagaStepHandler] | Fallback], ) -> None: @@ -105,11 +105,7 @@ async def load_completed_step_names(self) -> set[str]: Set of step names that have been completed """ history = await self._storage.get_step_history(self._saga_id) - return { - e.step_name - for e in history - if e.status == SagaStepStatus.COMPLETED and e.action == "act" - } + return {e.step_name for e in history if e.status == SagaStepStatus.COMPLETED and e.action == "act"} async def reconstruct_completed_steps( self, diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index 64d00e7..a461ea4 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -16,7 +16,7 @@ from cqrs.saga.models import ContextT from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.enums import SagaStatus, SagaStepStatus -from cqrs.saga.storage.protocol import ISagaStorage +from cqrs.saga.storage.protocol import ISagaStorage, SagaStorageRun from cqrs.saga.validation import ( SagaContextTypeExtractor, SagaStepValidator, @@ -160,69 +160,101 @@ async def __aiter__( This ensures data consistency and prevents "zombie states" where a saga is partially compensated and partially executed. - Strategy Overview: - - Forward Execution (RUNNING/PENDING): Execute steps sequentially, skipping - already completed steps for idempotency. - - Point of No Return: If saga status is COMPENSATING or FAILED, immediately - resume compensation without attempting forward execution. This prevents - inconsistent states where partial compensation conflicts with new execution. - - Local Retries: Retry logic is handled at the step level (within step.act()). - While retrying, saga status remains RUNNING, allowing recovery to continue - forward execution if the retry succeeds. - - Global Failure: Once all local retries are exhausted and saga transitions - to COMPENSATING, the path forward is permanently closed. Only compensation - can proceed. - - Yields: - SagaStepResult for each successfully executed step. - - Raises: - Exception: If any step fails, compensation is triggered and - the exception is re-raised. Also raised when recovering - a saga in COMPENSATING/FAILED status. + When storage supports create_run(), uses one session per saga and checkpoint + commits (fewer commits and sessions). Otherwise uses the legacy path. """ - # 1. Initialization / Recovery + try: + run_cm = self._storage.create_run() + except NotImplementedError: + run_cm = None + + if run_cm is not None: + async with run_cm as run: + async for step_result in self._execute(run): + yield step_result + else: + async for step_result in self._execute(None): + yield step_result + + async def _execute( + self, + run: SagaStorageRun | None, + ) -> typing.AsyncIterator[SagaStepResult[ContextT, typing.Any]]: + """Run saga steps; use run for storage when provided and commit at checkpoints.""" + if run is not None: + state_manager = SagaStateManager(self._saga_id, run) + recovery_manager = SagaRecoveryManager( + self._saga_id, + run, + self._container, + self._saga.steps, + ) + step_executor = SagaStepExecutor( + self._context, + self._container, + state_manager, + ) + fallback_executor = FallbackStepExecutor( + self._context, + self._container, + state_manager, + ) + compensator = SagaCompensator( + self._saga_id, + self._context, + run, + self._compensator._retry_count, + self._compensator._retry_delay, + self._compensator._retry_backoff, + on_after_compensate_step=run.commit, + ) + else: + state_manager = self._state_manager + recovery_manager = self._recovery_manager + step_executor = self._step_executor + fallback_executor = self._fallback_executor + compensator = self._compensator + completed_step_names: set[str] = set() if self._is_new_saga: - await self._state_manager.create_saga( + await state_manager.create_saga( self._saga.__class__.__name__, self._context, ) - await self._state_manager.update_status(SagaStatus.RUNNING) + await state_manager.update_status(SagaStatus.RUNNING) + if run is not None: + await run.commit() else: - # Try to recover state try: - status, _, _ = await self._storage.load_saga_state( - self._saga_id, - read_for_update=True, - ) + if run is not None: + status, _, _ = await run.load_saga_state( + self._saga_id, + read_for_update=True, + ) + else: + status, _, _ = await self._storage.load_saga_state( + self._saga_id, + read_for_update=True, + ) - # Check for terminal states first if status == SagaStatus.COMPLETED: logger.info( f"Saga {self._saga_id} is already {status}. Skipping execution.", ) return - # POINT OF NO RETURN: Strict Backward Recovery Strategy if status in (SagaStatus.COMPENSATING, SagaStatus.FAILED): logger.warning( f"Saga {self._saga_id} is in {status} state. " "Resuming compensation immediately.", ) - - # Restore completed steps from history for compensation - completed_act_steps = await self._recovery_manager.load_completed_step_names() - reconstructed_steps = await self._recovery_manager.reconstruct_completed_steps( + completed_act_steps = await recovery_manager.load_completed_step_names() + reconstructed_steps = await recovery_manager.reconstruct_completed_steps( completed_act_steps, ) - # Type cast is safe here because steps are reconstructed from the same saga - # that uses ContextT, so they have the correct context type - # We need to rebuild the list to satisfy type checker's invariance requirements self._completed_steps = [ typing.cast(SagaStepHandler[ContextT, typing.Any], step) for step in reconstructed_steps ] - if not self._completed_steps: logger.warning( f"Saga {self._saga_id}: no completed steps to compensate " @@ -230,47 +262,44 @@ async def __aiter__( "storage do not match saga step class names). " "Marking as FAILED without calling compensate().", ) - - # Immediately proceed to compensation - no forward execution - await self._compensate() - - # Raise exception to signal that saga was recovered in failed state + await compensator.compensate_steps(self._completed_steps) + if run is not None: + await run.commit() raise RuntimeError( f"Saga {self._saga_id} was recovered in {status} state " "and compensation was completed. Forward execution is not allowed.", ) - # For RUNNING/PENDING status, load history to skip completed steps - completed_step_names = await self._recovery_manager.load_completed_step_names() + completed_step_names = await recovery_manager.load_completed_step_names() except ValueError: - # If loading fails but ID was provided, create it - await self._state_manager.create_saga( + await state_manager.create_saga( self._saga.__class__.__name__, self._context, ) - await self._state_manager.update_status(SagaStatus.RUNNING) + await state_manager.update_status(SagaStatus.RUNNING) + if run is not None: + await run.commit() step_name = "unknown_step" try: for step_item in self._saga.steps: - # Check if this is a Fallback wrapper if isinstance(step_item, Fallback): ( step_result, executed_step, - ) = await self._fallback_executor.execute_fallback_step( + ) = await fallback_executor.execute_fallback_step( step_item, completed_step_names, ) if step_result is not None and executed_step is not None: - # Track completed step for compensation self._completed_steps.append(executed_step) + if run is not None: + await run.commit() yield dataclasses.replace( step_result, saga_id=self._saga_id, ) elif executed_step is None: - # Step was skipped (already completed), restore it for compensation primary_name = step_item.step.__name__ fallback_name = step_item.fallback.__name__ if primary_name in completed_step_names: @@ -281,47 +310,45 @@ async def __aiter__( self._completed_steps.append(step) continue - # Regular step handling step_type = step_item step_name = step_type.__name__ - # 2. Skip logic (Idempotency) if step_name in completed_step_names: - # Restore step instance to completed_steps for potential compensation step = await self._container.resolve(step_type) self._completed_steps.append(step) logger.debug(f"Skipping already completed step: {step_name}") continue - # 3. Execution - step_result = await self._step_executor.execute_step( + step_result = await step_executor.execute_step( step_type, step_name, ) - - # Track completed step for compensation step = await self._container.resolve(step_type) self._completed_steps.append(step) - + if run is not None: + await run.commit() yield dataclasses.replace( step_result, saga_id=self._saga_id, ) - # Update context one final time before marking as completed - await self._state_manager.update_context(self._context) - await self._state_manager.update_status(SagaStatus.COMPLETED) + await state_manager.update_context(self._context) + await state_manager.update_status(SagaStatus.COMPLETED) + if run is not None: + await run.commit() except Exception as e: - # Log failure for the specific step - await self._state_manager.log_step( + await state_manager.log_step( step_name, "act", SagaStepStatus.FAILED, str(e), ) self._error = e - await self._compensate() + self._compensated = True + await compensator.compensate_steps(self._completed_steps) + if run is not None: + await run.commit() raise async def _compensate(self) -> None: diff --git a/src/cqrs/saga/storage/__init__.py b/src/cqrs/saga/storage/__init__.py index f4dc64c..cfbd22f 100644 --- a/src/cqrs/saga/storage/__init__.py +++ b/src/cqrs/saga/storage/__init__.py @@ -1,10 +1,11 @@ from cqrs.saga.storage.enums import SagaStatus, SagaStepStatus from cqrs.saga.storage.models import SagaLogEntry -from cqrs.saga.storage.protocol import ISagaStorage +from cqrs.saga.storage.protocol import ISagaStorage, SagaStorageRun __all__ = [ "SagaStatus", "SagaStepStatus", "SagaLogEntry", "ISagaStorage", + "SagaStorageRun", ] diff --git a/src/cqrs/saga/storage/memory.py b/src/cqrs/saga/storage/memory.py index dfe851a..1c58e5e 100644 --- a/src/cqrs/saga/storage/memory.py +++ b/src/cqrs/saga/storage/memory.py @@ -1,3 +1,4 @@ +import contextlib import datetime import logging import typing @@ -6,11 +7,80 @@ from cqrs.dispatcher.exceptions import SagaConcurrencyError from cqrs.saga.storage.enums import SagaStatus, SagaStepStatus from cqrs.saga.storage.models import SagaLogEntry -from cqrs.saga.storage.protocol import ISagaStorage +from cqrs.saga.storage.protocol import ISagaStorage, SagaStorageRun logger = logging.getLogger("cqrs.saga.storage.memory") +class _MemorySagaStorageRun(SagaStorageRun): + """Run that delegates to the underlying MemorySagaStorage; commit/rollback are no-ops.""" + + def __init__(self, storage: "MemorySagaStorage") -> None: + self._storage = storage + + async def create_saga( + self, + saga_id: uuid.UUID, + name: str, + context: dict[str, typing.Any], + ) -> None: + await self._storage.create_saga(saga_id, name, context) + + async def update_context( + self, + saga_id: uuid.UUID, + context: dict[str, typing.Any], + current_version: int | None = None, + ) -> None: + await self._storage.update_context(saga_id, context, current_version) + + async def update_status( + self, + saga_id: uuid.UUID, + status: SagaStatus, + ) -> None: + await self._storage.update_status(saga_id, status) + + async def log_step( + self, + saga_id: uuid.UUID, + step_name: str, + action: typing.Literal["act", "compensate"], + status: SagaStepStatus, + details: str | None = None, + ) -> None: + await self._storage.log_step( + saga_id, + step_name, + action, + status, + details, + ) + + async def load_saga_state( + self, + saga_id: uuid.UUID, + *, + read_for_update: bool = False, + ) -> tuple[SagaStatus, dict[str, typing.Any], int]: + return await self._storage.load_saga_state( + saga_id, + read_for_update=read_for_update, + ) + + async def get_step_history( + self, + saga_id: uuid.UUID, + ) -> list[SagaLogEntry]: + return await self._storage.get_step_history(saga_id) + + async def commit(self) -> None: + pass + + async def rollback(self) -> None: + pass + + class MemorySagaStorage(ISagaStorage): """In-memory implementation of ISagaStorage for testing and development.""" @@ -20,6 +90,15 @@ def __init__(self) -> None: # Structure: {saga_id: [SagaLogEntry, ...]} self._logs: dict[uuid.UUID, list[SagaLogEntry]] = {} + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + @contextlib.asynccontextmanager + async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: + yield _MemorySagaStorageRun(self) + + return _run() + async def create_saga( self, saga_id: uuid.UUID, @@ -126,11 +205,7 @@ async def get_sagas_for_recovery( ) -> list[uuid.UUID]: recoverable = (SagaStatus.RUNNING, SagaStatus.COMPENSATING) now = datetime.datetime.now(datetime.timezone.utc) - threshold = ( - (now - datetime.timedelta(seconds=stale_after_seconds)) - if stale_after_seconds is not None - else None - ) + threshold = (now - datetime.timedelta(seconds=stale_after_seconds)) if stale_after_seconds is not None else None candidates = [ sid for sid, data in self._sagas.items() diff --git a/src/cqrs/saga/storage/protocol.py b/src/cqrs/saga/storage/protocol.py index 789ddde..e80faaf 100644 --- a/src/cqrs/saga/storage/protocol.py +++ b/src/cqrs/saga/storage/protocol.py @@ -1,4 +1,5 @@ import abc +import contextlib import typing import uuid @@ -6,6 +7,52 @@ from cqrs.saga.storage.models import SagaLogEntry +class SagaStorageRun(typing.Protocol): + """Protocol for a scoped saga storage run (one session, checkpoint commits). + + Returned by ISagaStorage.create_run(). Methods do not commit; the caller + must call commit() at checkpoints. Session is never exposed. + """ + + async def create_saga( + self, + saga_id: uuid.UUID, + name: str, + context: dict[str, typing.Any], + ) -> None: ... + async def update_context( + self, + saga_id: uuid.UUID, + context: dict[str, typing.Any], + current_version: int | None = None, + ) -> None: ... + async def update_status( + self, + saga_id: uuid.UUID, + status: SagaStatus, + ) -> None: ... + async def log_step( + self, + saga_id: uuid.UUID, + step_name: str, + action: typing.Literal["act", "compensate"], + status: SagaStepStatus, + details: str | None = None, + ) -> None: ... + async def load_saga_state( + self, + saga_id: uuid.UUID, + *, + read_for_update: bool = False, + ) -> tuple[SagaStatus, dict[str, typing.Any], int]: ... + async def get_step_history( + self, + saga_id: uuid.UUID, + ) -> list[SagaLogEntry]: ... + async def commit(self) -> None: ... + async def rollback(self) -> None: ... + + class ISagaStorage(abc.ABC): """Interface for saga persistence storage. @@ -214,3 +261,23 @@ async def set_recovery_attempts( attempts: The value to set recovery_attempts to (e.g. 0 to reset, or max_recovery_attempts to exclude from recovery). """ + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """Create a scoped run (one session) for saga execution with checkpoint commits. + + Optional. When implemented, the orchestrator uses one session per saga + and calls commit() only at checkpoints (after create+RUNNING, after each + step, at completion, during compensation). Reduces commits and sessions. + The yielded object has the same mutation/read methods as this storage + but does not commit; caller must call commit() or rollback(). + + Returns: + Async context manager yielding a run object (SagaStorageRun). + Session is never exposed outside the storage implementation. + + Raises: + NotImplementedError: If this storage does not support scoped runs. + """ + raise NotImplementedError("This storage does not support create_run()") diff --git a/src/cqrs/saga/storage/sqlalchemy.py b/src/cqrs/saga/storage/sqlalchemy.py index 83e8cd3..30ea014 100644 --- a/src/cqrs/saga/storage/sqlalchemy.py +++ b/src/cqrs/saga/storage/sqlalchemy.py @@ -1,3 +1,4 @@ +import contextlib import datetime import logging import os @@ -14,7 +15,7 @@ from cqrs.dispatcher.exceptions import SagaConcurrencyError from cqrs.saga.storage.enums import SagaStatus, SagaStepStatus from cqrs.saga.storage.models import SagaLogEntry -from cqrs.saga.storage.protocol import ISagaStorage +from cqrs.saga.storage.protocol import ISagaStorage, SagaStorageRun Base = registry().generate_base() logger = logging.getLogger(__name__) @@ -132,10 +133,168 @@ class SagaLogModel(Base): ) +class _SqlAlchemySagaStorageRun(SagaStorageRun): + """Scoped run: one session, no commit inside methods; caller calls commit().""" + + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def create_saga( + self, + saga_id: uuid.UUID, + name: str, + context: dict[str, typing.Any], + ) -> None: + execution = SagaExecutionModel( + id=saga_id, + name=name, + status=SagaStatus.PENDING, + context=context, + version=1, + recovery_attempts=0, + ) + self._session.add(execution) + + async def update_context( + self, + saga_id: uuid.UUID, + context: dict[str, typing.Any], + current_version: int | None = None, + ) -> None: + stmt = sqlalchemy.update(SagaExecutionModel).where( + SagaExecutionModel.id == saga_id, + ) + if current_version is not None: + stmt = stmt.where(SagaExecutionModel.version == current_version) + stmt = stmt.values( + context=context, + version=SagaExecutionModel.version + 1, + ) + else: + stmt = stmt.values( + context=context, + version=SagaExecutionModel.version + 1, + ) + result = await self._session.execute(stmt) + if result.rowcount == 0: # type: ignore[attr-defined] + if current_version is not None: + check_stmt = sqlalchemy.select(SagaExecutionModel.id).where( + SagaExecutionModel.id == saga_id, + ) + check_result = await self._session.execute(check_stmt) + if check_result.scalar_one_or_none(): + raise SagaConcurrencyError( + f"Saga {saga_id} was modified concurrently", + ) + raise SagaConcurrencyError( + f"Saga {saga_id} was modified concurrently or does not exist", + ) + + async def update_status( + self, + saga_id: uuid.UUID, + status: SagaStatus, + ) -> None: + await self._session.execute( + sqlalchemy.update(SagaExecutionModel) + .where(SagaExecutionModel.id == saga_id) + .values( + status=status, + version=SagaExecutionModel.version + 1, + ), + ) + + async def log_step( + self, + saga_id: uuid.UUID, + step_name: str, + action: typing.Literal["act", "compensate"], + status: SagaStepStatus, + details: str | None = None, + ) -> None: + log_entry = SagaLogModel( + saga_id=saga_id, + step_name=step_name, + action=action, + status=status, + details=details, + ) + self._session.add(log_entry) + + async def load_saga_state( + self, + saga_id: uuid.UUID, + *, + read_for_update: bool = False, + ) -> tuple[SagaStatus, dict[str, typing.Any], int]: + stmt = sqlalchemy.select(SagaExecutionModel).where( + SagaExecutionModel.id == saga_id, + ) + if read_for_update: + stmt = stmt.with_for_update() + result = await self._session.execute(stmt) + execution = result.scalars().first() + if not execution: + raise ValueError(f"Saga {saga_id} not found") + status_value: SagaStatus = typing.cast(SagaStatus, execution.status) + context_value: dict[str, typing.Any] = typing.cast( + dict[str, typing.Any], + execution.context, + ) + version_value: int = typing.cast(int, execution.version) + return status_value, context_value, version_value + + async def get_step_history( + self, + saga_id: uuid.UUID, + ) -> list[SagaLogEntry]: + result = await self._session.execute( + sqlalchemy.select(SagaLogModel).where(SagaLogModel.saga_id == saga_id).order_by(SagaLogModel.created_at), + ) + rows = result.scalars().all() + return [ + SagaLogEntry( + saga_id=typing.cast(uuid.UUID, row.saga_id), + step_name=typing.cast(str, row.step_name), + action=typing.cast(typing.Literal["act", "compensate"], row.action), + status=typing.cast(SagaStepStatus, row.status), + timestamp=typing.cast( + datetime.datetime, + row.created_at.replace(tzinfo=datetime.timezone.utc) + if row.created_at.tzinfo is None + else row.created_at, + ), + details=typing.cast(str | None, row.details), + ) + for row in rows + ] + + async def commit(self) -> None: + await self._session.commit() + + async def rollback(self) -> None: + await self._session.rollback() + + class SqlAlchemySagaStorage(ISagaStorage): def __init__(self, session_factory: async_sessionmaker[AsyncSession]): self.session_factory = session_factory + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + @contextlib.asynccontextmanager + async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: + async with self.session_factory() as session: + run = _SqlAlchemySagaStorageRun(session) + try: + yield run + except Exception: + await run.rollback() + raise + + return _run() + async def create_saga( self, saga_id: uuid.UUID, @@ -357,9 +516,7 @@ async def increment_recovery_attempts( if new_status is not None: values["status"] = new_status result = await session.execute( - sqlalchemy.update(SagaExecutionModel) - .where(SagaExecutionModel.id == saga_id) - .values(**values), + sqlalchemy.update(SagaExecutionModel).where(SagaExecutionModel.id == saga_id).values(**values), ) if result.rowcount == 0: # type: ignore[attr-defined] raise ValueError(f"Saga {saga_id} not found") diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index 71a14b3..814c29a 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -1,6 +1,11 @@ -"""Benchmarks for Saga with memory storage (dataclass DCResponse).""" +"""Benchmarks for Saga with memory storage (dataclass DCResponse). + +- Benchmarks named *_run_* use the scoped run path (create_run, checkpoint commits). +- Benchmarks named *_legacy_* use the legacy path (no create_run, commit per storage call). +""" import asyncio +import contextlib import dataclasses import typing @@ -11,6 +16,7 @@ from cqrs.saga.saga import Saga from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.memory import MemorySagaStorage +from cqrs.saga.storage.protocol import SagaStorageRun @dataclasses.dataclass @@ -141,6 +147,20 @@ def memory_storage() -> MemorySagaStorage: return MemorySagaStorage() +class MemorySagaStorageLegacy(MemorySagaStorage): + """Memory storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") + + +@pytest.fixture +def memory_storage_legacy() -> MemorySagaStorageLegacy: + return MemorySagaStorageLegacy() + + @pytest.fixture def saga_with_memory_storage( saga_container: SagaContainer, @@ -153,13 +173,13 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_memory_full_transaction( +def test_benchmark_saga_memory_run_full_transaction( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, memory_storage: MemorySagaStorage, ): - """Benchmark full saga transaction with memory storage (3 steps).""" + """Benchmark full saga transaction with memory storage, scoped run (3 steps).""" async def run() -> None: context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -175,13 +195,13 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_single_step( +def test_benchmark_saga_memory_run_single_step( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, memory_storage: MemorySagaStorage, ): - """Benchmark saga with single step (memory storage).""" + """Benchmark saga with single step, scoped run (memory storage).""" class SingleStepSaga(Saga[OrderContext]): steps = [ReserveInventoryStep] @@ -202,13 +222,13 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_ten_transactions( +def test_benchmark_saga_memory_run_ten_transactions( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, memory_storage: MemorySagaStorage, ): - """Benchmark 10 saga transactions in sequence (memory storage).""" + """Benchmark 10 saga transactions in sequence, scoped run (memory storage).""" async def run() -> None: for i in range(10): @@ -227,3 +247,82 @@ async def run() -> None: pass benchmark(lambda: asyncio.run(run())) + + +# ---- Legacy path (no create_run, commit per storage call) ---- + + +@pytest.mark.benchmark +def test_benchmark_saga_memory_legacy_full_transaction( + benchmark, + saga_with_memory_storage: Saga[OrderContext], + saga_container: SagaContainer, + memory_storage_legacy: MemorySagaStorageLegacy, +): + """Benchmark full saga transaction with memory storage, legacy path (3 steps).""" + + async def run() -> None: + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + async with saga_with_memory_storage.transaction( + context=context, + container=saga_container, + storage=memory_storage_legacy, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: asyncio.run(run())) + + +@pytest.mark.benchmark +def test_benchmark_saga_memory_legacy_single_step( + benchmark, + saga_with_memory_storage: Saga[OrderContext], + saga_container: SagaContainer, + memory_storage_legacy: MemorySagaStorageLegacy, +): + """Benchmark saga with single step, legacy path (memory storage).""" + + class SingleStepSaga(Saga[OrderContext]): + steps = [ReserveInventoryStep] + + saga = SingleStepSaga() + + async def run() -> None: + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + async with saga.transaction( + context=context, + container=saga_container, + storage=memory_storage_legacy, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: asyncio.run(run())) + + +@pytest.mark.benchmark +def test_benchmark_saga_memory_legacy_ten_transactions( + benchmark, + saga_with_memory_storage: Saga[OrderContext], + saga_container: SagaContainer, +): + """Benchmark 10 saga transactions in sequence, legacy path (memory storage).""" + + async def run() -> None: + for i in range(10): + storage = MemorySagaStorageLegacy() + context = OrderContext( + order_id=f"ord_{i}", + user_id=f"user_{i}", + amount=100.0 + i, + ) + async with saga_with_memory_storage.transaction( + context=context, + container=saga_container, + storage=storage, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: asyncio.run(run())) diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index f70e24f..0d6b432 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -1,9 +1,16 @@ -"""Benchmarks for Saga with SQLAlchemy storage (dataclass DCResponse). Requires DATABASE_DSN.""" +"""Benchmarks for Saga with SQLAlchemy storage (dataclass DCResponse). Requires DATABASE_DSN. + +- Benchmarks named *_run_* use the scoped run path (create_run, checkpoint commits). +- Benchmarks named *_legacy_* use the legacy path (no create_run, commit per storage call). +""" + +import contextlib import pytest from sqlalchemy.ext.asyncio import async_sessionmaker from cqrs.saga.saga import Saga +from cqrs.saga.storage.protocol import SagaStorageRun from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from .test_benchmark_saga_memory import ( @@ -15,6 +22,15 @@ ) +class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): + """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") + + @pytest.fixture def saga_container() -> SagaContainer: container = SagaContainer() @@ -33,13 +49,13 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_full_transaction( +def test_benchmark_saga_sqlalchemy_run_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, saga_benchmark_loop_and_engine, ): - """Benchmark full saga transaction with SQLAlchemy storage (MySQL).""" + """Benchmark full saga transaction with SQLAlchemy storage, scoped run (MySQL).""" loop, engine = saga_benchmark_loop_and_engine session_factory = async_sessionmaker( @@ -64,12 +80,12 @@ async def run_transaction() -> None: @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_single_step( +def test_benchmark_saga_sqlalchemy_run_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, ): - """Benchmark saga with single step (SQLAlchemy storage).""" + """Benchmark saga with single step, scoped run (SQLAlchemy storage).""" loop, engine = saga_benchmark_loop_and_engine class SingleStepSaga(Saga[OrderContext]): @@ -96,3 +112,72 @@ async def run_transaction() -> None: pass benchmark(lambda: loop.run_until_complete(run_transaction())) + + +# ---- Legacy path (no create_run, commit per storage call) ---- + + +@pytest.mark.benchmark +def test_benchmark_saga_sqlalchemy_legacy_full_transaction( + benchmark, + saga_sqlalchemy: Saga[OrderContext], + saga_container: SagaContainer, + saga_benchmark_loop_and_engine, +): + """Benchmark full saga transaction with SQLAlchemy storage, legacy path (MySQL).""" + loop, engine = saga_benchmark_loop_and_engine + + session_factory = async_sessionmaker( + engine, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + storage = SqlAlchemySagaStorageLegacy(session_factory) + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + + async def run_transaction() -> None: + async with saga_sqlalchemy.transaction( + context=context, + container=saga_container, + storage=storage, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: loop.run_until_complete(run_transaction())) + + +@pytest.mark.benchmark +def test_benchmark_saga_sqlalchemy_legacy_single_step( + benchmark, + saga_container: SagaContainer, + saga_benchmark_loop_and_engine, +): + """Benchmark saga with single step, legacy path (SQLAlchemy storage).""" + loop, engine = saga_benchmark_loop_and_engine + + class SingleStepSaga(Saga[OrderContext]): + steps = [ReserveInventoryStep] + + saga = SingleStepSaga() + + session_factory = async_sessionmaker( + engine, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + storage = SqlAlchemySagaStorageLegacy(session_factory) + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + + async def run_transaction() -> None: + async with saga.transaction( + context=context, + container=saga_container, + storage=storage, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: loop.run_until_complete(run_transaction())) diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index 66c90a9..c5ab5f2 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -1,6 +1,11 @@ -"""Benchmarks for Saga with memory storage (default Response).""" +"""Benchmarks for Saga with memory storage (default Response). + +- Benchmarks named *_run_* use the scoped run path (create_run, checkpoint commits). +- Benchmarks named *_legacy_* use the legacy path (no create_run, commit per storage call). +""" import asyncio +import contextlib import dataclasses import typing @@ -11,6 +16,7 @@ from cqrs.saga.saga import Saga from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.memory import MemorySagaStorage +from cqrs.saga.storage.protocol import SagaStorageRun @dataclasses.dataclass @@ -138,6 +144,20 @@ def memory_storage() -> MemorySagaStorage: return MemorySagaStorage() +class MemorySagaStorageLegacy(MemorySagaStorage): + """Memory storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") + + +@pytest.fixture +def memory_storage_legacy() -> MemorySagaStorageLegacy: + return MemorySagaStorageLegacy() + + @pytest.fixture def saga_with_memory_storage( saga_container: SagaContainer, @@ -150,13 +170,13 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_memory_full_transaction( +def test_benchmark_saga_memory_run_full_transaction( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, memory_storage: MemorySagaStorage, ): - """Benchmark full saga transaction with memory storage (3 steps).""" + """Benchmark full saga transaction with memory storage, scoped run (3 steps).""" async def run() -> None: context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -172,13 +192,13 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_single_step( +def test_benchmark_saga_memory_run_single_step( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, memory_storage: MemorySagaStorage, ): - """Benchmark saga with single step (memory storage).""" + """Benchmark saga with single step, scoped run (memory storage).""" class SingleStepSaga(Saga[OrderContext]): steps = [ReserveInventoryStep] @@ -199,13 +219,13 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_ten_transactions( +def test_benchmark_saga_memory_run_ten_transactions( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, memory_storage: MemorySagaStorage, ): - """Benchmark 10 saga transactions in sequence (memory storage).""" + """Benchmark 10 saga transactions in sequence, scoped run (memory storage).""" async def run() -> None: for i in range(10): @@ -224,3 +244,82 @@ async def run() -> None: pass benchmark(lambda: asyncio.run(run())) + + +# ---- Legacy path (no create_run, commit per storage call) ---- + + +@pytest.mark.benchmark +def test_benchmark_saga_memory_legacy_full_transaction( + benchmark, + saga_with_memory_storage: Saga[OrderContext], + saga_container: SagaContainer, + memory_storage_legacy: MemorySagaStorageLegacy, +): + """Benchmark full saga transaction with memory storage, legacy path (3 steps).""" + + async def run() -> None: + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + async with saga_with_memory_storage.transaction( + context=context, + container=saga_container, + storage=memory_storage_legacy, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: asyncio.run(run())) + + +@pytest.mark.benchmark +def test_benchmark_saga_memory_legacy_single_step( + benchmark, + saga_with_memory_storage: Saga[OrderContext], + saga_container: SagaContainer, + memory_storage_legacy: MemorySagaStorageLegacy, +): + """Benchmark saga with single step, legacy path (memory storage).""" + + class SingleStepSaga(Saga[OrderContext]): + steps = [ReserveInventoryStep] + + saga = SingleStepSaga() + + async def run() -> None: + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + async with saga.transaction( + context=context, + container=saga_container, + storage=memory_storage_legacy, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: asyncio.run(run())) + + +@pytest.mark.benchmark +def test_benchmark_saga_memory_legacy_ten_transactions( + benchmark, + saga_with_memory_storage: Saga[OrderContext], + saga_container: SagaContainer, +): + """Benchmark 10 saga transactions in sequence, legacy path (memory storage).""" + + async def run() -> None: + for i in range(10): + storage = MemorySagaStorageLegacy() + context = OrderContext( + order_id=f"ord_{i}", + user_id=f"user_{i}", + amount=100.0 + i, + ) + async with saga_with_memory_storage.transaction( + context=context, + container=saga_container, + storage=storage, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: asyncio.run(run())) diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index a3a4dd1..af151f5 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -1,9 +1,16 @@ -"""Benchmarks for Saga with SQLAlchemy storage (default Response). Requires DATABASE_DSN.""" +"""Benchmarks for Saga with SQLAlchemy storage (default Response). Requires DATABASE_DSN. + +- Benchmarks named *_run_* use the scoped run path (create_run, checkpoint commits). +- Benchmarks named *_legacy_* use the legacy path (no create_run, commit per storage call). +""" + +import contextlib import pytest from sqlalchemy.ext.asyncio import async_sessionmaker from cqrs.saga.saga import Saga +from cqrs.saga.storage.protocol import SagaStorageRun from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from .test_benchmark_saga_memory import ( @@ -15,6 +22,15 @@ ) +class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): + """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") + + @pytest.fixture def saga_container() -> SagaContainer: container = SagaContainer() @@ -33,13 +49,13 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_full_transaction( +def test_benchmark_saga_sqlalchemy_run_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, saga_benchmark_loop_and_engine, ): - """Benchmark full saga transaction with SQLAlchemy storage (MySQL).""" + """Benchmark full saga transaction with SQLAlchemy storage, scoped run (MySQL).""" loop, engine = saga_benchmark_loop_and_engine session_factory = async_sessionmaker( @@ -64,12 +80,12 @@ async def run_transaction() -> None: @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_single_step( +def test_benchmark_saga_sqlalchemy_run_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, ): - """Benchmark saga with single step (SQLAlchemy storage).""" + """Benchmark saga with single step, scoped run (SQLAlchemy storage).""" loop, engine = saga_benchmark_loop_and_engine class SingleStepSaga(Saga[OrderContext]): @@ -96,3 +112,72 @@ async def run_transaction() -> None: pass benchmark(lambda: loop.run_until_complete(run_transaction())) + + +# ---- Legacy path (no create_run, commit per storage call) ---- + + +@pytest.mark.benchmark +def test_benchmark_saga_sqlalchemy_legacy_full_transaction( + benchmark, + saga_sqlalchemy: Saga[OrderContext], + saga_container: SagaContainer, + saga_benchmark_loop_and_engine, +): + """Benchmark full saga transaction with SQLAlchemy storage, legacy path (MySQL).""" + loop, engine = saga_benchmark_loop_and_engine + + session_factory = async_sessionmaker( + engine, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + storage = SqlAlchemySagaStorageLegacy(session_factory) + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + + async def run_transaction() -> None: + async with saga_sqlalchemy.transaction( + context=context, + container=saga_container, + storage=storage, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: loop.run_until_complete(run_transaction())) + + +@pytest.mark.benchmark +def test_benchmark_saga_sqlalchemy_legacy_single_step( + benchmark, + saga_container: SagaContainer, + saga_benchmark_loop_and_engine, +): + """Benchmark saga with single step, legacy path (SQLAlchemy storage).""" + loop, engine = saga_benchmark_loop_and_engine + + class SingleStepSaga(Saga[OrderContext]): + steps = [ReserveInventoryStep] + + saga = SingleStepSaga() + + session_factory = async_sessionmaker( + engine, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + storage = SqlAlchemySagaStorageLegacy(session_factory) + context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) + + async def run_transaction() -> None: + async with saga.transaction( + context=context, + container=saga_container, + storage=storage, + ) as transaction: + async for _ in transaction: + pass + + benchmark(lambda: loop.run_until_complete(run_transaction())) diff --git a/tests/unit/test_saga/test_saga_storage_run.py b/tests/unit/test_saga/test_saga_storage_run.py new file mode 100644 index 0000000..c61c32d --- /dev/null +++ b/tests/unit/test_saga/test_saga_storage_run.py @@ -0,0 +1,211 @@ +"""Tests for saga storage create_run() and checkpoint (run) path.""" + +import typing +import uuid + +import pytest + +from cqrs.saga.storage.enums import SagaStatus, SagaStepStatus +from cqrs.saga.storage.memory import MemorySagaStorage +from cqrs.saga.storage.protocol import ISagaStorage + +from .conftest import ( + FailingStep, + OrderContext, + ReserveInventoryStep, + SagaContainer, + ShipOrderStep, +) + + +class StorageWithoutCreateRun(ISagaStorage): + """Storage that does not implement create_run (legacy path).""" + + def __init__(self) -> None: + self._inner = MemorySagaStorage() + + async def create_saga(self, saga_id: uuid.UUID, name: str, context: dict) -> None: + await self._inner.create_saga(saga_id, name, context) + + async def update_context( + self, + saga_id: uuid.UUID, + context: dict, + current_version: int | None = None, + ) -> None: + await self._inner.update_context(saga_id, context, current_version) + + async def update_status(self, saga_id: uuid.UUID, status: SagaStatus) -> None: + await self._inner.update_status(saga_id, status) + + async def log_step( + self, + saga_id: uuid.UUID, + step_name: str, + action: typing.Literal["act", "compensate"], + status: SagaStepStatus, + details: str | None = None, + ) -> None: + await self._inner.log_step(saga_id, step_name, action, status, details) + + async def load_saga_state( + self, + saga_id: uuid.UUID, + *, + read_for_update: bool = False, + ) -> tuple[SagaStatus, dict, int]: + return await self._inner.load_saga_state( + saga_id, + read_for_update=read_for_update, + ) + + async def get_step_history(self, saga_id: uuid.UUID) -> list: + return await self._inner.get_step_history(saga_id) + + async def get_sagas_for_recovery( + self, + limit: int, + max_recovery_attempts: int = 5, + stale_after_seconds: int | None = None, + saga_name: str | None = None, + ) -> list[uuid.UUID]: + return await self._inner.get_sagas_for_recovery( + limit, + max_recovery_attempts=max_recovery_attempts, + stale_after_seconds=stale_after_seconds, + saga_name=saga_name, + ) + + async def increment_recovery_attempts( + self, + saga_id: uuid.UUID, + new_status: SagaStatus | None = None, + ) -> None: + await self._inner.increment_recovery_attempts(saga_id, new_status) + + async def set_recovery_attempts(self, saga_id: uuid.UUID, attempts: int) -> None: + await self._inner.set_recovery_attempts(saga_id, attempts) + + +async def test_memory_storage_create_run_yields_run_with_required_methods() -> None: + """create_run() yields an object with create_saga, update_*, log_step, commit, rollback.""" + storage = MemorySagaStorage() + async with storage.create_run() as run: + assert run is not None + assert hasattr(run, "create_saga") + assert hasattr(run, "update_context") + assert hasattr(run, "update_status") + assert hasattr(run, "log_step") + assert hasattr(run, "load_saga_state") + assert hasattr(run, "get_step_history") + assert hasattr(run, "commit") + assert hasattr(run, "rollback") + + +async def test_memory_storage_run_commit_rollback_are_no_op() -> None: + """Run from MemorySagaStorage: commit and rollback do not raise.""" + storage = MemorySagaStorage() + async with storage.create_run() as run: + await run.commit() + await run.rollback() + + +async def test_memory_storage_run_persists_after_commit() -> None: + """Using run: create_saga + commit makes state visible to storage.""" + storage = MemorySagaStorage() + saga_id = uuid.uuid4() + async with storage.create_run() as run: + await run.create_saga(saga_id, "TestSaga", {"key": "value"}) + await run.update_status(saga_id, SagaStatus.RUNNING) + await run.commit() + status, context, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.RUNNING + assert context == {"key": "value"} + assert version >= 1 + + +async def test_saga_with_storage_with_create_run_completes_successfully() -> None: + """Saga with MemorySagaStorage (has create_run) uses run path and completes.""" + from cqrs.saga.saga import Saga + + class TwoStepSaga(Saga[OrderContext]): + steps = [ReserveInventoryStep, ShipOrderStep] + + container = SagaContainer() + container.register(ReserveInventoryStep, ReserveInventoryStep()) + container.register(ShipOrderStep, ShipOrderStep()) + storage = MemorySagaStorage() + saga = TwoStepSaga() + context = OrderContext(order_id="o1", user_id="u1", amount=50.0) + results = [] + async with saga.transaction( + context=context, + container=container, # type: ignore[arg-type] + storage=storage, + ) as transaction: + async for step_result in transaction: + results.append(step_result) + assert len(results) == 2 + status, _, _ = await storage.load_saga_state(transaction.saga_id) + assert status == SagaStatus.COMPLETED + + +async def test_saga_with_storage_without_create_run_completes_successfully() -> None: + """Saga with storage that does not implement create_run uses legacy path.""" + from cqrs.saga.saga import Saga + + class TwoStepSaga(Saga[OrderContext]): + steps = [ReserveInventoryStep, ShipOrderStep] + + container = SagaContainer() + container.register(ReserveInventoryStep, ReserveInventoryStep()) + container.register(ShipOrderStep, ShipOrderStep()) + storage = StorageWithoutCreateRun() + saga = TwoStepSaga() + context = OrderContext(order_id="o2", user_id="u2", amount=60.0) + results = [] + async with saga.transaction( + context=context, + container=container, # type: ignore[arg-type] + storage=storage, + ) as transaction: + async for step_result in transaction: + results.append(step_result) + assert len(results) == 2 + status, _, _ = await storage.load_saga_state(transaction.saga_id) + assert status == SagaStatus.COMPLETED + + +async def test_saga_with_run_path_compensates_on_failure() -> None: + """When a step fails, compensation runs and saga ends in FAILED (run path).""" + from cqrs.saga.saga import Saga + + class SagaWithFailure(Saga[OrderContext]): + steps = [ReserveInventoryStep, FailingStep] + + container = SagaContainer() + container.register(ReserveInventoryStep, ReserveInventoryStep()) + container.register(FailingStep, FailingStep()) + storage = MemorySagaStorage() + saga = SagaWithFailure() + context = OrderContext(order_id="o3", user_id="u3", amount=70.0) + saga_id: uuid.UUID | None = None + with pytest.raises(ValueError, match="Step failed"): + async with saga.transaction( + context=context, + container=container, # type: ignore[arg-type] + storage=storage, + ) as transaction: + saga_id = transaction.saga_id + async for _ in transaction: + pass + assert saga_id is not None + status, _, _ = await storage.load_saga_state(saga_id) + assert status == SagaStatus.FAILED + + +async def test_storage_create_run_raises_not_implemented_by_default() -> None: + """Default create_run() on a minimal storage raises NotImplementedError.""" + storage = StorageWithoutCreateRun() + with pytest.raises(NotImplementedError, match="does not support create_run"): + storage.create_run() From b92e39f4a545e37075d58fc53b0793be23ac5ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Wed, 11 Feb 2026 00:40:10 +0300 Subject: [PATCH 02/17] Increase version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e2fea71..3ff60e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ maintainers = [{name = "Vadim Kozyrevskiy", email = "vadikko2@mail.ru"}] name = "python-cqrs" readme = "README.md" requires-python = ">=3.10" -version = "4.8.1" +version = "4.9.0" [project.optional-dependencies] aiobreaker = ["aiobreaker>=0.3.0"] From a0eae83ab71ef2455269e73445d51806f6fa21e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Wed, 11 Feb 2026 00:47:20 +0300 Subject: [PATCH 03/17] Rename banchmarks --- tests/benchmarks/dataclasses/test_benchmark_saga_memory.py | 4 ++-- .../benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py | 4 ++-- tests/benchmarks/default/test_benchmark_saga_memory.py | 4 ++-- tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index 814c29a..b8ba894 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -173,7 +173,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_memory_run_full_transaction( +def test_benchmark_saga_memory_full_transaction( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, @@ -195,7 +195,7 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_run_single_step( +def test_benchmark_saga_memory_single_step( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index 0d6b432..232458c 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -49,7 +49,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_run_full_transaction( +def test_benchmark_saga_sqlalchemy_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, @@ -80,7 +80,7 @@ async def run_transaction() -> None: @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_run_single_step( +def test_benchmark_saga_sqlalchemy_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index c5ab5f2..db727c4 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -170,7 +170,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_memory_run_full_transaction( +def test_benchmark_saga_memory_full_transaction( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, @@ -192,7 +192,7 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_run_single_step( +def test_benchmark_saga_memory_single_step( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index af151f5..f5ca1cc 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -49,7 +49,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_run_full_transaction( +def test_benchmark_saga_sqlalchemy_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, @@ -80,7 +80,7 @@ async def run_transaction() -> None: @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_run_single_step( +def test_benchmark_saga_sqlalchemy_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, From 799753338d63b98775c9f3ee14bc9e209fe61a89 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 15:34:50 +0300 Subject: [PATCH 04/17] =?UTF-8?q?=F0=9F=93=9D=20Add=20docstrings=20to=20`r?= =?UTF-8?q?educe-saga-storage-overhead`=20(#62)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/saga.py | 23 +++- examples/saga_fallback.py | 20 ++- examples/saga_recovery.py | 66 ++++++---- examples/saga_recovery_scheduler.py | 8 +- examples/saga_sqlalchemy_storage.py | 7 +- src/cqrs/saga/compensation.py | 36 +++--- src/cqrs/saga/execution.py | 44 +++---- src/cqrs/saga/saga.py | 33 +++-- src/cqrs/saga/storage/memory.py | 114 ++++++++++++++++- src/cqrs/saga/storage/protocol.py | 107 +++++++++++++--- src/cqrs/saga/storage/sqlalchemy.py | 120 +++++++++++++++++- .../dataclasses/test_benchmark_saga_memory.py | 57 ++++++++- .../test_benchmark_saga_sqlalchemy.py | 41 +++++- .../default/test_benchmark_saga_memory.py | 57 ++++++++- .../default/test_benchmark_saga_sqlalchemy.py | 32 ++++- tests/unit/test_saga/test_saga_storage_run.py | 83 +++++++++++- 16 files changed, 734 insertions(+), 114 deletions(-) diff --git a/examples/saga.py b/examples/saga.py index b4956ce..a0710d6 100644 --- a/examples/saga.py +++ b/examples/saga.py @@ -275,7 +275,20 @@ async def create_shipment( items: list[str], address: str, ) -> tuple[str, str]: - """Create a shipment for the order.""" + """ + Create a shipment for an order and record its tracking number. + + Parameters: + order_id (str): Identifier of the order to ship. + items (list[str]): List of item identifiers included in the shipment. + address (str): Shipping address; must not be empty. + + Returns: + tuple[str, str]: A tuple containing the created `shipment_id` and its `tracking_number`. + + Raises: + ValueError: If `address` is empty. + """ if not address: raise ValueError("Shipping address is required") @@ -469,7 +482,11 @@ class OrderSaga(Saga[OrderContext]): async def run_successful_saga() -> None: - """Demonstrate a successful saga execution.""" + """ + Run an example order-processing saga and print the per-step progress and final results. + + Sets up mock services, dependency injection, and in-memory saga storage; executes the OrderSaga with a generated saga ID, prints each completed step, then prints the final saga status, context fields (inventory reservation, payment ID, shipment ID) and the persisted execution log. If saga execution fails, the failure is printed and the exception is re-raised. + """ print("\n" + "=" * 70) print("SCENARIO 1: Successful Order Processing Saga") print("=" * 70) @@ -736,4 +753,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/examples/saga_fallback.py b/examples/saga_fallback.py index 407b162..dd95fa8 100644 --- a/examples/saga_fallback.py +++ b/examples/saga_fallback.py @@ -144,7 +144,17 @@ async def act( self, context: OrderContext, ) -> SagaStepResult[OrderContext, ReserveInventoryResponse]: - """Primary step that always raises an error.""" + """ + Simulate a failing primary reservation step for the saga. + + This action always raises a RuntimeError to emulate an unavailable downstream service and trigger fallback or compensation behavior. + + Parameters: + context (OrderContext): Shared saga context containing order details (e.g., order_id, user_id, amount, reservation_id). + + Raises: + RuntimeError: Indicates the primary step failed (service unavailable). + """ self._call_count += 1 logger.info( f" [PrimaryStep] Executing act() for order {context.order_id} " f"(call #{self._call_count})...", @@ -271,7 +281,11 @@ async def run_saga( async def main() -> None: - """Run saga fallback example.""" + """ + Run an interactive demonstration of the saga fallback pattern with a circuit breaker. + + Executes three scenarios that show a failing primary step with an automatic fallback, the circuit breaker opening after a configurable number of failures, and fail-fast behavior when the circuit is open. Also conditionally demonstrates configuring a Redis-backed circuit breaker storage, prints per-scenario results and a summary, and informs about missing optional dependencies. + """ print("\n" + "=" * 80) print("SAGA FALLBACK PATTERN WITH CIRCUIT BREAKER EXAMPLE") print("=" * 80) @@ -419,4 +433,4 @@ class OrderSagaWithRedisBreaker(Saga[OrderContext]): if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/examples/saga_recovery.py b/examples/saga_recovery.py index 72dfd7b..3400c94 100644 --- a/examples/saga_recovery.py +++ b/examples/saga_recovery.py @@ -281,7 +281,20 @@ async def create_shipment( items: list[str], address: str, ) -> tuple[str, str]: - """Create a shipment for the order.""" + """ + Create a shipment record for the given order and generate a tracking number. + + Parameters: + order_id (str): Identifier of the order. + items (list[str]): Items included in the shipment. + address (str): Destination shipping address. + + Returns: + tuple[str, str]: A tuple containing the shipment ID and the tracking number. + + Raises: + ValueError: If `address` is empty. + """ if not address: raise ValueError("Shipping address is required") @@ -519,16 +532,12 @@ async def resolve(self, type_: type) -> typing.Any: async def simulate_interrupted_saga() -> tuple[uuid.UUID, MemorySagaStorage]: """ - Simulate a saga that gets interrupted after the first step. - - This simulates what happens when: - - Server crashes after completing ReserveInventoryStep - - Network timeout occurs - - Process is killed during execution - - Database connection is lost - + Simulate a saga that is interrupted after the inventory reservation step to produce a recoverable persisted state. + Returns: - Tuple of (saga_id, storage) so we can recover it later. + tuple: + saga_id (uuid.UUID): Identifier of the created saga. + storage (MemorySagaStorage): In-memory storage containing the persisted saga state and step history for recovery. """ print("\n" + "=" * 70) print("SCENARIO 1: Simulating Interrupted Saga") @@ -622,13 +631,13 @@ async def recover_interrupted_saga( storage: MemorySagaStorage, ) -> None: """ - Recover and complete the interrupted saga. - - This demonstrates how recovery ensures eventual consistency by: - 1. Loading saga state from storage - 2. Reconstructing context - 3. Resuming execution from last completed step - 4. Completing remaining steps + Recover and complete an interrupted saga using persisted state. + + Loads the saga state from storage, reconstructs the saga context, resumes execution from the last completed step, and completes any remaining steps to restore eventual consistency. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga instance to recover. + storage (MemorySagaStorage): Durable storage containing the saga's persisted state and step history. """ print("\n" + "=" * 70) print("SCENARIO 2: Recovering Interrupted Saga") @@ -688,10 +697,12 @@ async def recover_interrupted_saga( async def simulate_interrupted_compensation() -> tuple[uuid.UUID, MemorySagaStorage]: """ - Simulate a saga that fails and gets interrupted during compensation. - - This shows recovery of compensation logic, which is critical for - maintaining consistency when rollback is interrupted. + Simulate a saga that fails and is interrupted during compensation. + + Sets up services, a saga, and a failing shipment step to trigger compensation that is then artificially interrupted; returns identifiers and storage state for performing recovery in a separate run. + + Returns: + tuple[uuid.UUID, MemorySagaStorage]: The saga ID and the in-memory storage containing the persisted saga state and step history after the simulated interruption. """ print("\n" + "=" * 70) print("SCENARIO 3: Simulating Interrupted Compensation") @@ -801,10 +812,13 @@ async def recover_interrupted_compensation( storage: MemorySagaStorage, ) -> None: """ - Recover and complete the interrupted compensation. - - This ensures that even if compensation is interrupted, it will - eventually complete, releasing all resources. + Recover and complete an interrupted compensation for a saga. + + Loads the saga state from the provided storage using the given saga identifier and drives any incomplete compensation steps to completion, ensuring resources (inventory, payments, shipments) are released and the system reaches a consistent state. Progress and final status are printed to stdout. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to recover. + storage (MemorySagaStorage): Persistent storage containing the saga state and step history. """ print("\n" + "=" * 70) print("SCENARIO 4: Recovering Interrupted Compensation") @@ -910,4 +924,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/examples/saga_recovery_scheduler.py b/examples/saga_recovery_scheduler.py index a3f3103..e36d4f6 100644 --- a/examples/saga_recovery_scheduler.py +++ b/examples/saga_recovery_scheduler.py @@ -620,7 +620,11 @@ async def create_interrupted_saga(storage: MemorySagaStorage) -> uuid.UUID: async def main() -> None: - """Run the recovery scheduler example.""" + """ + Run the saga recovery scheduler demo and display its outcome. + + Sets up an in-memory saga storage, creates a simulated interrupted saga and marks it stale, runs the recovery loop for three iterations (using the module's recovery_loop and recovery configuration constants), then loads and prints the final saga state. + """ print("\n" + "=" * 70) print("SAGA RECOVERY SCHEDULER EXAMPLE") print("=" * 70) @@ -660,4 +664,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/examples/saga_sqlalchemy_storage.py b/examples/saga_sqlalchemy_storage.py index 2daa543..d600026 100644 --- a/examples/saga_sqlalchemy_storage.py +++ b/examples/saga_sqlalchemy_storage.py @@ -142,6 +142,11 @@ async def setup_database(engine: AsyncEngine) -> None: async def main() -> None: # 1. Create SQLAlchemy Engine with Connection Pool # SQLAlchemy creates a pool by default (QueuePool for most dialects, SingletonThreadPool for SQLite) + """ + Run a demonstration that executes an OrderSaga using an async SQLAlchemy engine and persistent SqlAlchemySagaStorage. + + Initializes a pooled async SQLAlchemy engine and schema, creates a session factory and SqlAlchemySagaStorage, bootstraps a mediator with a DI container and saga mapper, runs an OrderSaga while streaming step results to stdout, and then reloads and prints the persisted saga state and step history before disposing the engine. + """ engine = create_async_engine( DB_URL, echo=False, # Set to True to see SQL queries @@ -202,4 +207,4 @@ def saga_mapper(mapper: cqrs.SagaMap) -> None: if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/src/cqrs/saga/compensation.py b/src/cqrs/saga/compensation.py index 6b0280a..53dd1ec 100644 --- a/src/cqrs/saga/compensation.py +++ b/src/cqrs/saga/compensation.py @@ -26,17 +26,16 @@ def __init__( on_after_compensate_step: typing.Callable[[], typing.Awaitable[None]] | None = None, ) -> None: """ - Initialize compensator. - - Args: - saga_id: UUID of the saga - context: Saga context - storage: Saga storage implementation (or run object with same interface) - retry_count: Number of retry attempts for compensation - retry_delay: Initial delay between retries in seconds - retry_backoff: Backoff multiplier for exponential delay - on_after_compensate_step: Optional async callback after each successfully - compensated step (e.g. run.commit() for checkpoint). + Create a SagaCompensator configured to perform compensation of completed saga steps with retry and optional post-step callback. + + Parameters: + saga_id: Identifier of the saga. + context: Saga execution context passed to step compensation handlers. + storage: Storage or run object implementing saga persistence operations. + retry_count: Maximum number of attempts per step before giving up. + retry_delay: Initial delay in seconds before the first retry. + retry_backoff: Multiplier applied to the delay for each successive retry (exponential backoff). + on_after_compensate_step: Optional async callback invoked after each step is successfully compensated. """ self._saga_id = saga_id self._context = context @@ -51,10 +50,15 @@ async def compensate_steps( completed_steps: list[SagaStepHandler[ContextT, typing.Any]], ) -> None: """ - Compensate all completed steps in reverse order with retry mechanism. - - Args: - completed_steps: List of completed step handlers to compensate + Compensates completed saga steps in reverse order, applying retry logic and recording step statuses. + + Compensates each handler from last to first, skipping steps already recorded as compensated in the saga history. Updates the saga status to COMPENSATING at the start and logs per-step statuses (STARTED, COMPLETED, FAILED) in storage. After a step completes, the optional on_after_compensate_step callback (if provided) is awaited. If any step fails after all retry attempts, the saga is marked as FAILED. If no completed steps are provided, no compensation is attempted and the saga is marked as FAILED. + + Parameters: + completed_steps (list[SagaStepHandler[ContextT, typing.Any]]): Handlers corresponding to steps that completed during the saga; these will be compensated in reverse order. + + Returns: + None """ await self._storage.update_status(self._saga_id, SagaStatus.COMPENSATING) @@ -170,4 +174,4 @@ async def _compensate_step_with_retry( # If we get here, all retries failed if last_exception: - raise last_exception + raise last_exception \ No newline at end of file diff --git a/src/cqrs/saga/execution.py b/src/cqrs/saga/execution.py index d5ca0f7..84a63bf 100644 --- a/src/cqrs/saga/execution.py +++ b/src/cqrs/saga/execution.py @@ -24,11 +24,11 @@ def __init__( storage: ISagaStorage | SagaStorageRun, ) -> None: """ - Initialize state manager. - - Args: - saga_id: UUID of the saga - storage: Saga storage implementation + Create a SagaStateManager bound to a specific saga identifier and storage backend. + + Parameters: + saga_id: Identifier for the saga instance. + storage: Storage backend implementing ISagaStorage or SagaStorageRun used to persist saga state and history. """ self._saga_id = saga_id self._storage = storage @@ -84,13 +84,13 @@ def __init__( saga_steps: list[type[SagaStepHandler] | Fallback], ) -> None: """ - Initialize recovery manager. - - Args: - saga_id: UUID of the saga - storage: Saga storage implementation - container: DI container for resolving step handlers - saga_steps: List of saga steps + Construct a SagaRecoveryManager that holds the identifiers, storage, DI container, and configured saga steps required to reconstruct a saga's execution state. + + Parameters: + saga_id: Identifier for the saga instance (e.g., UUID or other unique value). + storage: Persistence backend implementing saga history operations (ISagaStorage or SagaStorageRun). + container: Dependency injection container used to resolve step handler instances. + saga_steps: Ordered list of saga step types or Fallback wrappers that define the saga's execution sequence. """ self._saga_id = saga_id self._storage = storage @@ -99,10 +99,10 @@ def __init__( async def load_completed_step_names(self) -> set[str]: """ - Load set of completed step names from history. - + Return the names of saga steps that completed their primary ("act") action. + Returns: - Set of step names that have been completed + set[str]: Step names recorded with status `SagaStepStatus.COMPLETED` and action `"act"`. """ history = await self._storage.get_step_history(self._saga_id) return {e.step_name for e in history if e.status == SagaStepStatus.COMPLETED and e.action == "act"} @@ -112,13 +112,13 @@ async def reconstruct_completed_steps( completed_step_names: set[str], ) -> list[SagaStepHandler[SagaContext, typing.Any]]: """ - Reconstruct list of completed step handlers from history. - - Args: - completed_step_names: Set of completed step names - + Reconstructs and returns the resolved step handler instances corresponding to the completed steps, preserving saga execution order. + + Parameters: + completed_step_names (set[str]): Names of steps that completed the "act" action. + Returns: - List of step handlers in execution order + list[SagaStepHandler[SagaContext, typing.Any]]: Resolved step handler instances in execution order. For Fallback wrappers, the primary handler is chosen if its name appears in completed_step_names; otherwise the fallback handler is chosen when present. """ completed_steps: list[SagaStepHandler[SagaContext, typing.Any]] = [] @@ -365,4 +365,4 @@ async def execute_fallback_step( raise fallback_error else: # Should not fallback, re-raise original error - raise primary_error + raise primary_error \ No newline at end of file diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index a461ea4..2edd336 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -154,14 +154,10 @@ async def __aiter__( ) -> typing.AsyncIterator[SagaStepResult[ContextT, typing.Any]]: """ Execute saga steps sequentially and yield each step result. - - This method implements the "Strict Backward Recovery" strategy for saga execution. - Once a saga enters COMPENSATING or FAILED status, it can never proceed forward. - This ensures data consistency and prevents "zombie states" where a saga is - partially compensated and partially executed. - - When storage supports create_run(), uses one session per saga and checkpoint - commits (fewer commits and sessions). Otherwise uses the legacy path. + + Implements the Strict Backward Recovery strategy: if the saga is in COMPENSATING or FAILED status, forward execution is never resumed. When the underlying storage provides create_run(), execution is performed within a per-saga run with checkpoint commits; otherwise the legacy run-less path is used. + Returns: + AsyncIterator[SagaStepResult[ContextT, typing.Any]]: An async iterator that yields the result for each executed saga step in order. """ try: run_cm = self._storage.create_run() @@ -180,7 +176,18 @@ async def _execute( self, run: SagaStorageRun | None, ) -> typing.AsyncIterator[SagaStepResult[ContextT, typing.Any]]: - """Run saga steps; use run for storage when provided and commit at checkpoints.""" + """ + Execute the saga's configured steps, using the provided storage run for checkpointed operations when available, and perform recovery and compensation as required. + + Parameters: + run (SagaStorageRun | None): Optional per-saga storage run. When provided, the run is used for loading saga state, creating run-scoped managers/executors, and committing at checkpoint boundaries. When None, the transaction's internal managers and executors are used. + + Returns: + Async iterator that yields SagaStepResult values for each step that completes; each yielded result will include the transaction's saga_id. + + Raises: + RuntimeError: If the saga was recovered in COMPENSATING or FAILED state and compensation was completed, forward execution is not allowed. + """ if run is not None: state_manager = SagaStateManager(self._saga_id, run) recovery_manager = SagaRecoveryManager( @@ -352,7 +359,11 @@ async def _execute( raise async def _compensate(self) -> None: - """Compensate all completed steps in reverse order with retry mechanism.""" + """ + Mark the transaction as compensated and run compensation for all completed steps in reverse order. + + Sets an internal flag to prevent repeated compensation and delegates to the compensator which applies the configured retry behavior. + """ # Prevent double compensation if self._compensated: return @@ -479,4 +490,4 @@ def transaction( compensation_retry_count=compensation_retry_count, compensation_retry_delay=compensation_retry_delay, compensation_retry_backoff=compensation_retry_backoff, - ) + ) \ No newline at end of file diff --git a/src/cqrs/saga/storage/memory.py b/src/cqrs/saga/storage/memory.py index 1c58e5e..a1ec80f 100644 --- a/src/cqrs/saga/storage/memory.py +++ b/src/cqrs/saga/storage/memory.py @@ -16,6 +16,12 @@ class _MemorySagaStorageRun(SagaStorageRun): """Run that delegates to the underlying MemorySagaStorage; commit/rollback are no-ops.""" def __init__(self, storage: "MemorySagaStorage") -> None: + """ + Initialize the run and bind it to the provided MemorySagaStorage. + + Parameters: + storage (MemorySagaStorage): Underlying in-memory storage instance used to delegate saga operations. + """ self._storage = storage async def create_saga( @@ -24,6 +30,17 @@ async def create_saga( name: str, context: dict[str, typing.Any], ) -> None: + """ + Create a new saga entry in the underlying memory storage. + + Parameters: + saga_id (uuid.UUID): Unique identifier for the saga. + name (str): Human-readable saga name. + context (dict[str, typing.Any]): Initial saga context payload. + + Raises: + ValueError: If a saga with the same `saga_id` already exists. + """ await self._storage.create_saga(saga_id, name, context) async def update_context( @@ -32,6 +49,18 @@ async def update_context( context: dict[str, typing.Any], current_version: int | None = None, ) -> None: + """ + Update the stored context for the given saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose context will be updated. + context (dict[str, typing.Any]): New context to store for the saga. + current_version (int | None): If provided, require the stored saga version to match this value (optimistic locking). + + Raises: + ValueError: If the saga_id does not exist. + SagaConcurrencyError: If current_version is provided and does not match the stored version. + """ await self._storage.update_context(saga_id, context, current_version) async def update_status( @@ -39,6 +68,16 @@ async def update_status( saga_id: uuid.UUID, status: SagaStatus, ) -> None: + """ + Update the stored status of the saga identified by `saga_id`. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to update. + status (SagaStatus): New status to set for the saga. + + Raises: + ValueError: If no saga exists with the given `saga_id`. + """ await self._storage.update_status(saga_id, status) async def log_step( @@ -49,6 +88,16 @@ async def log_step( status: SagaStepStatus, details: str | None = None, ) -> None: + """ + Log a step entry for the given saga into the underlying storage. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga. + step_name (str): Name of the saga step. + action (Literal["act", "compensate"]): Whether the step is a forward action ("act") or a compensation ("compensate"). + status (SagaStepStatus): Outcome status of the step. + details (str | None): Optional free-form details or metadata about the step. + """ await self._storage.log_step( saga_id, step_name, @@ -63,6 +112,16 @@ async def load_saga_state( *, read_for_update: bool = False, ) -> tuple[SagaStatus, dict[str, typing.Any], int]: + """ + Load the current state for a saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to load. + read_for_update (bool): If True, acquire the state for update (may be used for optimistic locking or exclusive access). + + Returns: + tuple[SagaStatus, dict[str, typing.Any], int]: A tuple containing the saga's status, its context dictionary, and the current version number. + """ return await self._storage.load_saga_state( saga_id, read_for_update=read_for_update, @@ -72,12 +131,29 @@ async def get_step_history( self, saga_id: uuid.UUID, ) -> list[SagaLogEntry]: + """ + Retrieve the step log/history for a saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose step history is requested. + + Returns: + list[SagaLogEntry]: Saga step log entries sorted by timestamp in ascending order (oldest first). Returns an empty list if no logs exist. + """ return await self._storage.get_step_history(saga_id) async def commit(self) -> None: + """ + No-op commit for an in-memory saga run; provided to satisfy the SagaStorageRun interface. + + This method intentionally performs no action because the memory storage does not require an explicit commit. + """ pass async def rollback(self) -> None: + """ + Perform no action for rollback; provided to satisfy the SagaStorageRun interface. + """ pass @@ -86,6 +162,13 @@ class MemorySagaStorage(ISagaStorage): def __init__(self) -> None: # Structure: {saga_id: {name, status, context, created_at, updated_at, version}} + """ + Initialize in-memory storage for sagas and their step logs. + + Creates two internal mappings: + - _sagas: maps saga_id (UUID) to a dictionary containing keys `name`, `status`, `context`, `created_at`, `updated_at`, and `version`. + - _logs: maps saga_id (UUID) to a list of SagaLogEntry objects representing the saga's step history. + """ self._sagas: dict[uuid.UUID, dict[str, typing.Any]] = {} # Structure: {saga_id: [SagaLogEntry, ...]} self._logs: dict[uuid.UUID, list[SagaLogEntry]] = {} @@ -93,6 +176,12 @@ def __init__(self) -> None: def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """ + Provide an asynchronous context manager that yields a SagaStorageRun bound to this storage. + + Returns: + An asynchronous context manager that yields a `SagaStorageRun` instance backed by this `MemorySagaStorage`. + """ @contextlib.asynccontextmanager async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: yield _MemorySagaStorageRun(self) @@ -105,6 +194,17 @@ async def create_saga( name: str, context: dict[str, typing.Any], ) -> None: + """ + Create a new saga record in the in-memory store. + + Parameters: + saga_id (uuid.UUID): Identifier for the saga; must not already exist. + name (str): Human-readable name for the saga. + context (dict[str, typing.Any]): Initial context payload for the saga. + + Raises: + ValueError: If a saga with `saga_id` already exists. + """ if saga_id in self._sagas: raise ValueError(f"Saga {saga_id} already exists") @@ -203,6 +303,18 @@ async def get_sagas_for_recovery( stale_after_seconds: int | None = None, saga_name: str | None = None, ) -> list[uuid.UUID]: + """ + Selects saga IDs eligible for recovery based on status, recovery attempts, staleness, and an optional name filter. + + Parameters: + limit (int): Maximum number of saga IDs to return. + max_recovery_attempts (int): Upper bound (exclusive) on recovery attempts; only sagas with fewer attempts are considered. + stale_after_seconds (int | None): If provided, only sagas last updated earlier than this many seconds before now are considered; if None, staleness is ignored. + saga_name (str | None): If provided, only sagas with this name are considered; if None, name is not filtered. + + Returns: + list[uuid.UUID]: Up to `limit` saga IDs sorted by oldest `updated_at` first that match the recovery criteria. + """ recoverable = (SagaStatus.RUNNING, SagaStatus.COMPENSATING) now = datetime.datetime.now(datetime.timezone.utc) threshold = (now - datetime.timedelta(seconds=stale_after_seconds)) if stale_after_seconds is not None else None @@ -241,4 +353,4 @@ async def set_recovery_attempts( data = self._sagas[saga_id] data["recovery_attempts"] = attempts data["updated_at"] = datetime.datetime.now(datetime.timezone.utc) - data["version"] += 1 + data["version"] += 1 \ No newline at end of file diff --git a/src/cqrs/saga/storage/protocol.py b/src/cqrs/saga/storage/protocol.py index e80faaf..701259c 100644 --- a/src/cqrs/saga/storage/protocol.py +++ b/src/cqrs/saga/storage/protocol.py @@ -19,18 +19,48 @@ async def create_saga( saga_id: uuid.UUID, name: str, context: dict[str, typing.Any], - ) -> None: ... + ) -> None: """ + Create a new saga execution record with initial PENDING status and version 1. + + Parameters: + saga_id (uuid.UUID): Unique identifier for the saga (primary key). + name (str): Human-friendly name used for diagnostics and filtering. + context (dict[str, Any]): JSON-serializable initial saga context to persist. + """ + ... async def update_context( self, saga_id: uuid.UUID, context: dict[str, typing.Any], current_version: int | None = None, - ) -> None: ... + ) -> None: """ + Persist a snapshot of the saga's execution context, optionally using optimistic locking. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to update. + context (dict[str, Any]): JSON-serializable context object to store as the new snapshot. + current_version (int | None): If provided, perform an optimistic-locking update that succeeds only + if the stored version matches this value; on success the stored version is incremented. + + Raises: + SagaConcurrencyError: If `current_version` is provided and does not match the stored version. + """ + ... async def update_status( self, saga_id: uuid.UUID, status: SagaStatus, - ) -> None: ... + ) -> None: """ + Set the global status for the saga identified by `saga_id`. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to update. + status (SagaStatus): New global status to persist (for example RUNNING, COMPLETED, COMPENSATING). + + Notes: + This operation does not commit the storage session; the caller must call `commit()` on the active run or session to persist the change. + """ + ... async def log_step( self, saga_id: uuid.UUID, @@ -38,19 +68,58 @@ async def log_step( action: typing.Literal["act", "compensate"], status: SagaStepStatus, details: str | None = None, - ) -> None: ... + ) -> None: """ + Append a step transition to the saga's execution log. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose log will be appended. + step_name (str): Logical name of the step (used for diagnostics and replay). + action (Literal["act", "compensate"]): Whether this entry records the primary action ("act") or its compensating action ("compensate"). + status (SagaStepStatus): The step transition status to record (e.g., started, completed, failed, compensated). + details (str | None): Optional human-readable details or diagnostics about the transition. + """ + ... async def load_saga_state( self, saga_id: uuid.UUID, *, read_for_update: bool = False, - ) -> tuple[SagaStatus, dict[str, typing.Any], int]: ... + ) -> tuple[SagaStatus, dict[str, typing.Any], int]: """ + Load the current saga execution state. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to load. + read_for_update (bool): If True, acquire a database lock for update to prevent concurrent modifications. + + Returns: + tuple[SagaStatus, dict[str, Any], int]: A tuple containing the saga's global status, the latest persisted context (JSON-serializable), and the current optimistic-locking version number. + """ + ... async def get_step_history( self, saga_id: uuid.UUID, - ) -> list[SagaLogEntry]: ... - async def commit(self) -> None: ... - async def rollback(self) -> None: ... + ) -> list[SagaLogEntry]: """ + Retrieve the chronological step log for a saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose step history to retrieve. + + Returns: + list[SagaLogEntry]: Ordered list of step log entries for the saga, from oldest to newest. + """ + ... + async def commit(self) -> None: """ +Finalize the storage run by persisting and committing all pending changes made during this session. + +This method makes the run's checkpointed changes durable; the caller is responsible for invoking commit at logical checkpoints to persist session state. +""" +... + async def rollback(self) -> None: """ +Abort the current storage run and revert any uncommitted changes in the session. + +This releases the run's transactional state without persisting pending updates so that the storage remains as it was before the run began. +""" +... class ISagaStorage(abc.ABC): @@ -265,19 +334,15 @@ async def set_recovery_attempts( def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """Create a scoped run (one session) for saga execution with checkpoint commits. - - Optional. When implemented, the orchestrator uses one session per saga - and calls commit() only at checkpoints (after create+RUNNING, after each - step, at completion, during compensation). Reduces commits and sessions. - The yielded object has the same mutation/read methods as this storage - but does not commit; caller must call commit() or rollback(). - + """ + Create a scoped async run context for a single saga execution session with checkpointed commits. + + The context manager yields a SagaStorageRun that provides the same mutation/read methods as the storage but does not commit automatically; the caller must call commit() or rollback() at desired checkpoints. + Returns: - Async context manager yielding a run object (SagaStorageRun). - Session is never exposed outside the storage implementation. - + contextlib.AbstractAsyncContextManager[SagaStorageRun]: Async context manager yielding a SagaStorageRun session. + Raises: - NotImplementedError: If this storage does not support scoped runs. + NotImplementedError: If the storage backend does not support scoped runs. """ - raise NotImplementedError("This storage does not support create_run()") + raise NotImplementedError("This storage does not support create_run()") \ No newline at end of file diff --git a/src/cqrs/saga/storage/sqlalchemy.py b/src/cqrs/saga/storage/sqlalchemy.py index 30ea014..b85c870 100644 --- a/src/cqrs/saga/storage/sqlalchemy.py +++ b/src/cqrs/saga/storage/sqlalchemy.py @@ -137,6 +137,12 @@ class _SqlAlchemySagaStorageRun(SagaStorageRun): """Scoped run: one session, no commit inside methods; caller calls commit().""" def __init__(self, session: AsyncSession) -> None: + """ + Initialize the run wrapper with an async SQLAlchemy session. + + Parameters: + session (AsyncSession): The AsyncSession instance scoped to this run, used for all database operations. + """ self._session = session async def create_saga( @@ -145,6 +151,16 @@ async def create_saga( name: str, context: dict[str, typing.Any], ) -> None: + """ + Create and stage a new saga execution record in the current session with initial metadata. + + Creates a SagaExecutionModel for the given saga identifier with status set to PENDING, version set to 1, and recovery_attempts set to 0, and adds it to the active session without committing the transaction. + + Parameters: + saga_id (uuid.UUID): Unique identifier for the saga execution. + name (str): Human-readable name of the saga. + context (dict[str, Any]): Initial saga context to be stored (will be serialized to the model's JSON column). + """ execution = SagaExecutionModel( id=saga_id, name=name, @@ -161,6 +177,17 @@ async def update_context( context: dict[str, typing.Any], current_version: int | None = None, ) -> None: + """ + Update the stored context for a saga and increment its version, optionally enforcing an optimistic version check. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to update. + context (dict[str, typing.Any]): New serialized saga context to persist. + current_version (int | None): If provided, require the saga's current version to match this value before updating. + + Raises: + SagaConcurrencyError: If an optimistic version check fails (indicating a concurrent modification) or if the saga does not exist when a version was supplied. + """ stmt = sqlalchemy.update(SagaExecutionModel).where( SagaExecutionModel.id == saga_id, ) @@ -195,6 +222,16 @@ async def update_status( saga_id: uuid.UUID, status: SagaStatus, ) -> None: + """ + Update the stored status of a saga execution and increment its optimistic-lock version. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga execution to update. + status (SagaStatus): New status to set for the saga. + + Note: + The update is executed in the active database session; a commit is required to persist the change. + """ await self._session.execute( sqlalchemy.update(SagaExecutionModel) .where(SagaExecutionModel.id == saga_id) @@ -212,6 +249,16 @@ async def log_step( status: SagaStepStatus, details: str | None = None, ) -> None: + """ + Record a saga step event by creating and staging a log entry in the active session. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga execution. + step_name (str): Name of the step being recorded. + action (Literal["act", "compensate"]): The performed action: "act" for normal action or "compensate" for compensation. + status (SagaStepStatus): The step's outcome status. + details (str | None): Optional free-form details or error message associated with the step. + """ log_entry = SagaLogModel( saga_id=saga_id, step_name=step_name, @@ -227,6 +274,19 @@ async def load_saga_state( *, read_for_update: bool = False, ) -> tuple[SagaStatus, dict[str, typing.Any], int]: + """ + Load the current execution state for a saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to load. + read_for_update (bool): If true, acquire a row-level lock for update. + + Returns: + tuple[SagaStatus, dict[str, Any], int]: The saga's status, its context dictionary, and the current version. + + Raises: + ValueError: If no saga with the given id exists. + """ stmt = sqlalchemy.select(SagaExecutionModel).where( SagaExecutionModel.id == saga_id, ) @@ -248,6 +308,16 @@ async def get_step_history( self, saga_id: uuid.UUID, ) -> list[SagaLogEntry]: + """ + Retrieve chronological step log entries for the given saga. + + Parameters: + saga_id (uuid.UUID): UUID of the saga whose step history to fetch. + + Returns: + list[SagaLogEntry]: List of log entries ordered by creation time. Each entry's `timestamp` + is normalized to UTC if not already timezone-aware. + """ result = await self._session.execute( sqlalchemy.select(SagaLogModel).where(SagaLogModel.saga_id == saga_id).order_by(SagaLogModel.created_at), ) @@ -270,19 +340,42 @@ async def get_step_history( ] async def commit(self) -> None: + """ + Commit the current transaction in the associated AsyncSession. + """ await self._session.commit() async def rollback(self) -> None: + """ + Revert all staged changes in the current session's transaction. + + This aborts the in-progress transaction associated with the run's AsyncSession, + discarding any pending writes or flushes. + """ await self._session.rollback() class SqlAlchemySagaStorage(ISagaStorage): def __init__(self, session_factory: async_sessionmaker[AsyncSession]): + """ + Initialize the SQLAlchemy-based saga storage with a factory for creating async sessions. + + Parameters: + session_factory (async_sessionmaker[AsyncSession]): Factory that produces new AsyncSession instances used for each storage run and operation. + """ self.session_factory = session_factory def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """ + Create a scoped run that yields a SagaStorageRun bound to a fresh session. + + The returned context manager provides a run object whose lifecycle is tied to a single session. If an exception is raised inside the context, the run's transaction is rolled back; the session is always closed on exit. + + Returns: + A context manager that yields a `SagaStorageRun`. On exception within the context, the run's `rollback()` is invoked and the session is closed when the context exits. + """ @contextlib.asynccontextmanager async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: async with self.session_factory() as session: @@ -301,6 +394,20 @@ async def create_saga( name: str, context: dict[str, typing.Any], ) -> None: + """ + Create and persist a new saga execution record with initial metadata. + + Creates a SagaExecutionModel for the given saga_id and name, sets status to PENDING, + version to 1, and recovery_attempts to 0, and commits it to the database. + + Parameters: + saga_id (uuid.UUID): Unique identifier for the saga execution. + name (str): Human-readable saga name. + context (dict[str, typing.Any]): Initial saga context to store. + + Raises: + SQLAlchemyError: If the database operation fails; the transaction is rolled back before the exception is propagated. + """ async with self.session_factory() as session: try: execution = SagaExecutionModel( @@ -507,6 +614,17 @@ async def increment_recovery_attempts( saga_id: uuid.UUID, new_status: SagaStatus | None = None, ) -> None: + """ + Increment the recovery attempts counter for the given saga execution and optionally update its status. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga execution to update. + new_status (SagaStatus | None): If provided, set the saga's status to this value. + + Raises: + ValueError: If no saga execution exists with the given `saga_id`. + SQLAlchemyError: On database errors; the transaction is rolled back and the error is propagated. + """ async with self.session_factory() as session: try: values: dict[str, typing.Any] = { @@ -545,4 +663,4 @@ async def set_recovery_attempts( await session.commit() except SQLAlchemyError: await session.rollback() - raise + raise \ No newline at end of file diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index 814c29a..a236c56 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -144,6 +144,12 @@ def saga_container() -> SagaContainer: @pytest.fixture def memory_storage() -> MemorySagaStorage: + """ + Provide a fresh in-memory saga storage instance for tests and benchmarks. + + Returns: + MemorySagaStorage: A new MemorySagaStorage instance. + """ return MemorySagaStorage() @@ -153,11 +159,23 @@ class MemorySagaStorageLegacy(MemorySagaStorage): def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """ + Indicate that creating a scoped run context is not supported for the legacy in-memory storage used in benchmarks. + + Raises: + NotImplementedError: Always raised with message "Legacy storage: create_run disabled for benchmark". + """ raise NotImplementedError("Legacy storage: create_run disabled for benchmark") @pytest.fixture def memory_storage_legacy() -> MemorySagaStorageLegacy: + """ + Provide a legacy in-memory saga storage that does not support scoped runs. + + Returns: + MemorySagaStorageLegacy: An in-memory saga storage whose `create_run` is disabled (raises `NotImplementedError`) for legacy-path benchmarks. + """ return MemorySagaStorageLegacy() @@ -166,6 +184,18 @@ def saga_with_memory_storage( saga_container: SagaContainer, memory_storage: MemorySagaStorage, ) -> Saga[OrderContext]: + """ + Create an OrderSaga configured with reserve-inventory, process-payment, and ship-order steps. + + This factory accepts the saga container and memory storage as fixture dependencies (they are not used by this function) and returns a Saga subclass instance with the three ordered step handlers: ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep. + + Parameters: + saga_container (SagaContainer): Fixture-provided container (unused). + memory_storage (MemorySagaStorage): Fixture-provided memory storage (unused). + + Returns: + Saga[OrderContext]: An OrderSaga instance wired with the predefined steps. + """ class OrderSaga(Saga[OrderContext]): steps = [ReserveInventoryStep, ProcessPaymentStep, ShipOrderStep] @@ -182,6 +212,11 @@ def test_benchmark_saga_memory_run_full_transaction( """Benchmark full saga transaction with memory storage, scoped run (3 steps).""" async def run() -> None: + """ + Execute a full saga transaction using the module's memory-backed saga, advancing through every step. + + Creates an OrderContext with order_id "ord_1", user_id "user_1", and amount 100.0, opens a transaction using the provided saga container and memory storage, and iterates the transaction to exercise each step in the run path. + """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async with saga_with_memory_storage.transaction( context=context, @@ -231,6 +266,11 @@ def test_benchmark_saga_memory_run_ten_transactions( """Benchmark 10 saga transactions in sequence, scoped run (memory storage).""" async def run() -> None: + """ + Execute 10 sequential saga transactions using a fresh in-memory storage and context for each iteration. + + Each iteration creates a new MemorySagaStorage and OrderContext, opens a saga transaction with the provided container and storage, and iterates through the transaction steps without performing additional work. + """ for i in range(10): storage = MemorySagaStorage() context = OrderContext( @@ -262,6 +302,11 @@ def test_benchmark_saga_memory_legacy_full_transaction( """Benchmark full saga transaction with memory storage, legacy path (3 steps).""" async def run() -> None: + """ + Run a full-order saga transaction against the legacy in-memory storage used for benchmarks. + + Creates an OrderContext and executes the saga transaction using the provided saga container and legacy memory storage, iterating the transaction to completion. + """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async with saga_with_memory_storage.transaction( context=context, @@ -289,6 +334,11 @@ class SingleStepSaga(Saga[OrderContext]): saga = SingleStepSaga() async def run() -> None: + """ + Run a saga transaction using the legacy memory storage and iterate its steps. + + Enters a transaction for an OrderContext (order_id "ord_1") with the registered saga_container and memory_storage_legacy, then iterates through the transaction steps without performing work for each step. + """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async with saga.transaction( context=context, @@ -310,6 +360,11 @@ def test_benchmark_saga_memory_legacy_ten_transactions( """Benchmark 10 saga transactions in sequence, legacy path (memory storage).""" async def run() -> None: + """ + Execute ten sequential saga transactions using MemorySagaStorageLegacy. + + Each iteration creates a new MemorySagaStorageLegacy and an OrderContext (with distinct order_id, user_id, and amount) and runs the configured saga transaction to completion by iterating through its steps. + """ for i in range(10): storage = MemorySagaStorageLegacy() context = OrderContext( @@ -325,4 +380,4 @@ async def run() -> None: async for _ in transaction: pass - benchmark(lambda: asyncio.run(run())) + benchmark(lambda: asyncio.run(run())) \ No newline at end of file diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index 0d6b432..8e43f7a 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -28,11 +28,23 @@ class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """ + Disable creation of a scoped SagaStorageRun for legacy storage used in benchmarks. + + Raises: + NotImplementedError: Always raised with message "Legacy storage: create_run disabled for benchmark". + """ raise NotImplementedError("Legacy storage: create_run disabled for benchmark") @pytest.fixture def saga_container() -> SagaContainer: + """ + Create and return a SagaContainer pre-registered with the ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep instances. + + Returns: + SagaContainer: A container with the three steps already registered. + """ container = SagaContainer() container.register(ReserveInventoryStep, ReserveInventoryStep()) container.register(ProcessPaymentStep, ProcessPaymentStep()) @@ -103,6 +115,11 @@ class SingleStepSaga(Saga[OrderContext]): context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: + """ + Execute the saga transaction lifecycle by entering the saga's transaction context and iterating its steps to completion. + + This function opens the saga transaction using the surrounding `saga`, `saga_container`, `storage`, and `context`, then consumes the transaction iterator to drive all saga steps to completion. + """ async with saga.transaction( context=context, container=saga_container, @@ -124,7 +141,11 @@ def test_benchmark_saga_sqlalchemy_legacy_full_transaction( saga_container: SagaContainer, saga_benchmark_loop_and_engine, ): - """Benchmark full saga transaction with SQLAlchemy storage, legacy path (MySQL).""" + """ + Benchmark a full saga transaction using SQLAlchemy storage in legacy mode. + + Runs a complete saga (three-step OrderSaga) against SqlAlchemySagaStorageLegacy, which disables `create_run` so the storage exercises the legacy commit-per-call path. The benchmark executes the saga transaction in the provided event loop and database engine fixture. + """ loop, engine = saga_benchmark_loop_and_engine session_factory = async_sessionmaker( @@ -137,6 +158,11 @@ def test_benchmark_saga_sqlalchemy_legacy_full_transaction( context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: + """ + Execute the configured saga transaction and iterate through all its steps to completion. + + This coroutine opens a transaction using the surrounding `saga_sqlalchemy`, `saga_container`, `context`, and `storage` variables and consumes the transaction iterator without performing additional actions. + """ async with saga_sqlalchemy.transaction( context=context, container=saga_container, @@ -154,7 +180,11 @@ def test_benchmark_saga_sqlalchemy_legacy_single_step( saga_container: SagaContainer, saga_benchmark_loop_and_engine, ): - """Benchmark saga with single step, legacy path (SQLAlchemy storage).""" + """ + Benchmark executing a single-step Saga using legacy SQLAlchemy storage (commit-per-call path). + + Constructs a SingleStepSaga with ReserveInventoryStep, creates a SqlAlchemySagaStorageLegacy backed by the provided engine/session factory, and measures running a full saga transaction (iterating the transaction to completion) using the provided event loop via the benchmark fixture. + """ loop, engine = saga_benchmark_loop_and_engine class SingleStepSaga(Saga[OrderContext]): @@ -172,6 +202,11 @@ class SingleStepSaga(Saga[OrderContext]): context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: + """ + Execute the saga transaction lifecycle by entering the saga's transaction context and iterating its steps to completion. + + This function opens the saga transaction using the surrounding `saga`, `saga_container`, `storage`, and `context`, then consumes the transaction iterator to drive all saga steps to completion. + """ async with saga.transaction( context=context, container=saga_container, @@ -180,4 +215,4 @@ async def run_transaction() -> None: async for _ in transaction: pass - benchmark(lambda: loop.run_until_complete(run_transaction())) + benchmark(lambda: loop.run_until_complete(run_transaction())) \ No newline at end of file diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index c5ab5f2..5d207ba 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -141,6 +141,12 @@ def saga_container() -> SagaContainer: @pytest.fixture def memory_storage() -> MemorySagaStorage: + """ + Create a fresh in-memory saga storage instance for tests. + + Returns: + MemorySagaStorage: A new MemorySagaStorage used to persist saga state in memory. + """ return MemorySagaStorage() @@ -150,11 +156,26 @@ class MemorySagaStorageLegacy(MemorySagaStorage): def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """ + Disable creation of scoped storage runs for the legacy storage variant used in benchmarks. + + Returns: + contextlib.AbstractAsyncContextManager[SagaStorageRun]: An async context manager yielding a `SagaStorageRun` (disabled in this legacy implementation). + + Raises: + NotImplementedError: Always raised with the message "Legacy storage: create_run disabled for benchmark". + """ raise NotImplementedError("Legacy storage: create_run disabled for benchmark") @pytest.fixture def memory_storage_legacy() -> MemorySagaStorageLegacy: + """ + Create a MemorySagaStorageLegacy instance for legacy-path benchmarks. + + Returns: + MemorySagaStorageLegacy: A storage instance where `create_run()` is disabled and will raise NotImplementedError if called. + """ return MemorySagaStorageLegacy() @@ -163,6 +184,12 @@ def saga_with_memory_storage( saga_container: SagaContainer, memory_storage: MemorySagaStorage, ) -> Saga[OrderContext]: + """ + Create an OrderSaga preconfigured with inventory reservation, payment processing, and shipping steps. + + Returns: + Saga[OrderContext]: An instance configured with ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep. + """ class OrderSaga(Saga[OrderContext]): steps = [ReserveInventoryStep, ProcessPaymentStep, ShipOrderStep] @@ -179,6 +206,11 @@ def test_benchmark_saga_memory_run_full_transaction( """Benchmark full saga transaction with memory storage, scoped run (3 steps).""" async def run() -> None: + """ + Execute a full three-step OrderSaga transaction using the memory storage scoped-run path. + + Creates an OrderContext and runs the saga transaction to completion with the provided saga container and memory storage. + """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async with saga_with_memory_storage.transaction( context=context, @@ -228,6 +260,14 @@ def test_benchmark_saga_memory_run_ten_transactions( """Benchmark 10 saga transactions in sequence, scoped run (memory storage).""" async def run() -> None: + """ + Run ten sequential saga transactions, each using a new MemorySagaStorage and an OrderContext. + + Each iteration (i from 0 to 9) creates: + - a fresh MemorySagaStorage, + - an OrderContext with order_id "ord_i", user_id "user_i", and amount 100.0 + i, + then opens a transaction from `saga_with_memory_storage` with `saga_container` and the storage and iterates the transaction to completion. + """ for i in range(10): storage = MemorySagaStorage() context = OrderContext( @@ -259,6 +299,11 @@ def test_benchmark_saga_memory_legacy_full_transaction( """Benchmark full saga transaction with memory storage, legacy path (3 steps).""" async def run() -> None: + """ + Execute a full OrderSaga transaction using the legacy memory storage path. + + Builds an OrderContext (order_id "ord_1", user_id "user_1", amount 100.0) and runs the saga_with_memory_storage transaction with saga_container and memory_storage_legacy, iterating the transaction to completion. + """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async with saga_with_memory_storage.transaction( context=context, @@ -286,6 +331,11 @@ class SingleStepSaga(Saga[OrderContext]): saga = SingleStepSaga() async def run() -> None: + """ + Runs a full OrderSaga transaction using the legacy memory storage path. + + This coroutine executes the saga with an OrderContext and the MemorySagaStorageLegacy instance so the saga proceeds through all steps while exercising the legacy storage behavior (create_run disabled, commit-per-call path). + """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async with saga.transaction( context=context, @@ -307,6 +357,11 @@ def test_benchmark_saga_memory_legacy_ten_transactions( """Benchmark 10 saga transactions in sequence, legacy path (memory storage).""" async def run() -> None: + """ + Run ten sequential saga transactions using the legacy memory storage path. + + For each iteration this function creates a new MemorySagaStorageLegacy, constructs an OrderContext with a unique order_id and user_id and increasing amount, opens a saga transaction using the shared saga_with_memory_storage and saga_container, and iterates the transaction to completion. + """ for i in range(10): storage = MemorySagaStorageLegacy() context = OrderContext( @@ -322,4 +377,4 @@ async def run() -> None: async for _ in transaction: pass - benchmark(lambda: asyncio.run(run())) + benchmark(lambda: asyncio.run(run())) \ No newline at end of file diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index af151f5..b750a7f 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -28,11 +28,26 @@ class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): def create_run( self, ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """ + Disable scoped run creation for legacy storage used in benchmarks. + + This storage intentionally does not provide a scoped `create_run` context manager. + Calling this method raises a NotImplementedError to indicate the legacy path is in use. + + Raises: + NotImplementedError: Always raised to indicate scoped run creation is disabled for legacy storage. + """ raise NotImplementedError("Legacy storage: create_run disabled for benchmark") @pytest.fixture def saga_container() -> SagaContainer: + """ + Create a SagaContainer pre-registered with the standard order saga steps. + + Returns: + SagaContainer: Container with ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep registered. + """ container = SagaContainer() container.register(ReserveInventoryStep, ReserveInventoryStep()) container.register(ProcessPaymentStep, ProcessPaymentStep()) @@ -103,6 +118,11 @@ class SingleStepSaga(Saga[OrderContext]): context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: + """ + Run the saga transaction to completion by iterating over its yielded steps using the configured context, container, and storage. + + This function is used by benchmarks to execute a full saga flow without performing additional work per step. + """ async with saga.transaction( context=context, container=saga_container, @@ -137,6 +157,11 @@ def test_benchmark_saga_sqlalchemy_legacy_full_transaction( context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: + """ + Execute the saga transaction and iterate through all produced steps without performing any operations. + + Opens the saga's transaction context with the configured container and storage, then consumes every yielded step (no-op per step). Intended for benchmarking the transaction iteration path. + """ async with saga_sqlalchemy.transaction( context=context, container=saga_container, @@ -172,6 +197,11 @@ class SingleStepSaga(Saga[OrderContext]): context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: + """ + Run the saga transaction to completion by iterating over its yielded steps using the configured context, container, and storage. + + This function is used by benchmarks to execute a full saga flow without performing additional work per step. + """ async with saga.transaction( context=context, container=saga_container, @@ -180,4 +210,4 @@ async def run_transaction() -> None: async for _ in transaction: pass - benchmark(lambda: loop.run_until_complete(run_transaction())) + benchmark(lambda: loop.run_until_complete(run_transaction())) \ No newline at end of file diff --git a/tests/unit/test_saga/test_saga_storage_run.py b/tests/unit/test_saga/test_saga_storage_run.py index c61c32d..2e0c575 100644 --- a/tests/unit/test_saga/test_saga_storage_run.py +++ b/tests/unit/test_saga/test_saga_storage_run.py @@ -22,9 +22,20 @@ class StorageWithoutCreateRun(ISagaStorage): """Storage that does not implement create_run (legacy path).""" def __init__(self) -> None: + """ + Create a storage wrapper that delegates all saga operations to an internal in-memory storage while intentionally not providing `create_run`. + """ self._inner = MemorySagaStorage() async def create_saga(self, saga_id: uuid.UUID, name: str, context: dict) -> None: + """ + Create a new saga record with the given identifier, name, and initial context. + + Parameters: + saga_id (uuid.UUID): Unique identifier for the saga. + name (str): Human-readable name of the saga. + context (dict): Initial context payload for the saga. + """ await self._inner.create_saga(saga_id, name, context) async def update_context( @@ -33,9 +44,24 @@ async def update_context( context: dict, current_version: int | None = None, ) -> None: + """ + Update the stored context for a saga, optionally validating the expected current version. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose context will be updated. + context (dict): New context data to persist for the saga. + current_version (int | None): If provided, the update will only proceed when the stored version equals this value; pass None to skip version validation. + """ await self._inner.update_context(saga_id, context, current_version) async def update_status(self, saga_id: uuid.UUID, status: SagaStatus) -> None: + """ + Update the stored status of a saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to update. + status (SagaStatus): New status to set for the saga. + """ await self._inner.update_status(saga_id, status) async def log_step( @@ -46,6 +72,16 @@ async def log_step( status: SagaStepStatus, details: str | None = None, ) -> None: + """ + Record the execution or compensation outcome of a saga step. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga. + step_name (str): Name of the step being logged. + action (Literal["act", "compensate"]): Whether this log entry is for the step's normal action ("act") or its compensation ("compensate"). + status (SagaStepStatus): Resulting status of the step. + details (str | None): Optional human-readable details or metadata about the step event. + """ await self._inner.log_step(saga_id, step_name, action, status, details) async def load_saga_state( @@ -54,12 +90,31 @@ async def load_saga_state( *, read_for_update: bool = False, ) -> tuple[SagaStatus, dict, int]: + """ + Load the current state for a saga from the underlying storage. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga to load. + read_for_update (bool): If True, load the state with intent to update (may acquire locks or use a read-for-update strategy). + + Returns: + tuple[SagaStatus, dict, int]: A tuple containing the saga's status, its context dictionary, and the current version number. + """ return await self._inner.load_saga_state( saga_id, read_for_update=read_for_update, ) async def get_step_history(self, saga_id: uuid.UUID) -> list: + """ + Return the step execution history for the given saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose history to retrieve. + + Returns: + list: Step history records in chronological order. Each record describes the step name, action ("act" or "compensate"), step status, timestamp, and any optional details. + """ return await self._inner.get_step_history(saga_id) async def get_sagas_for_recovery( @@ -69,6 +124,18 @@ async def get_sagas_for_recovery( stale_after_seconds: int | None = None, saga_name: str | None = None, ) -> list[uuid.UUID]: + """ + Selects saga IDs that are eligible for recovery. + + Parameters: + limit (int): Maximum number of saga IDs to return. + max_recovery_attempts (int): Only include sagas with fewer than this many recovery attempts. + stale_after_seconds (int | None): If provided, only include sagas last updated more than this many seconds ago; if None, do not filter by staleness. + saga_name (str | None): If provided, restrict results to sagas with this name. + + Returns: + list[uuid.UUID]: Saga UUIDs that match the recovery criteria, up to `limit`. + """ return await self._inner.get_sagas_for_recovery( limit, max_recovery_attempts=max_recovery_attempts, @@ -81,9 +148,23 @@ async def increment_recovery_attempts( saga_id: uuid.UUID, new_status: SagaStatus | None = None, ) -> None: + """ + Increment the recovery-attempts counter for a saga and optionally update its status. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose recovery attempts should be incremented. + new_status (SagaStatus | None): If provided, update the saga's status to this value after incrementing attempts; otherwise leave status unchanged. + """ await self._inner.increment_recovery_attempts(saga_id, new_status) async def set_recovery_attempts(self, saga_id: uuid.UUID, attempts: int) -> None: + """ + Set the number of recovery attempts recorded for a saga. + + Parameters: + saga_id (uuid.UUID): Identifier of the saga whose recovery attempts will be set. + attempts (int): Number of recovery attempts to record; should be zero or a positive integer. + """ await self._inner.set_recovery_attempts(saga_id, attempts) @@ -208,4 +289,4 @@ async def test_storage_create_run_raises_not_implemented_by_default() -> None: """Default create_run() on a minimal storage raises NotImplementedError.""" storage = StorageWithoutCreateRun() with pytest.raises(NotImplementedError, match="does not support create_run"): - storage.create_run() + storage.create_run() \ No newline at end of file From cc4a0483cc4c3c85944bc672bb442030f136bc70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Thu, 19 Feb 2026 19:24:33 +0300 Subject: [PATCH 05/17] Fixes after review --- README.md | 6 +- examples/saga_recovery_scheduler.py | 28 +++--- examples/saga_sqlalchemy_storage.py | 6 +- src/cqrs/saga/compensation.py | 16 +++- src/cqrs/saga/saga.py | 85 +++++++++++++------ src/cqrs/saga/storage/protocol.py | 55 ++++++------ src/cqrs/saga/storage/sqlalchemy.py | 28 ++---- tests/benchmarks/__init__.py | 1 + .../dataclasses/test_benchmark_saga_memory.py | 36 ++------ .../test_benchmark_saga_sqlalchemy.py | 20 +---- .../default/test_benchmark_saga_memory.py | 23 +---- .../default/test_benchmark_saga_sqlalchemy.py | 23 +---- tests/benchmarks/storage_legacy.py | 29 +++++++ tests/unit/test_saga/test_saga_basic.py | 2 +- 14 files changed, 167 insertions(+), 191 deletions(-) create mode 100644 tests/benchmarks/storage_legacy.py diff --git a/README.md b/README.md index 5b09fe0..5863766 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ class JoinMeetingCommandHandler(RequestHandler[JoinMeetingCommand, None]): def __init__(self, meetings_api: MeetingAPIProtocol) -> None: self._meetings_api = meetings_api - self.events: list[Event] = [] + self._events: list[Event] = [] @property def events(self) -> typing.List[events.Event]: @@ -115,7 +115,7 @@ class ReadMeetingQueryHandler(RequestHandler[ReadMeetingQuery, ReadMeetingQueryR def __init__(self, meetings_api: MeetingAPIProtocol) -> None: self._meetings_api = meetings_api - self.events: list[Event] = [] + self._events: list[Event] = [] @property def events(self) -> typing.List[events.Event]: @@ -323,7 +323,7 @@ def init_queries(mapper: requests.RequestMap) -> None: mapper.bind(queries.ReadMeetingQuery, query_handlers.ReadMeetingQueryHandler) def init_events(mapper: events.EventMap) -> None: - mapper.bind(events.NotificationEvent[events_models.NotificationMeetingRoomClosed], event_handlers.MeetingRoomClosedNotificationHandler) + mapper.bind(events.NotificationEvent[event_models.NotificationMeetingRoomClosed], event_handlers.MeetingRoomClosedNotificationHandler) mapper.bind(events.NotificationEvent[event_models.ECSTMeetingRoomClosed], event_handlers.UpdateMeetingRoomReadModelHandler) ``` diff --git a/examples/saga_recovery_scheduler.py b/examples/saga_recovery_scheduler.py index e36d4f6..387ac13 100644 --- a/examples/saga_recovery_scheduler.py +++ b/examples/saga_recovery_scheduler.py @@ -79,6 +79,7 @@ import dataclasses import datetime import logging +import traceback import typing import uuid @@ -263,12 +264,7 @@ async def create_shipment( shipment_id = f"shipment_{order_id}" tracking_number = f"TRACK{self._tracking_counter:08d}" self._shipments[shipment_id] = tracking_number - logger.info( - " ✓ Created shipment %s for order %s (tracking: %s)", - shipment_id, - order_id, - tracking_number, - ) + logger.info(f" ✓ Created shipment {shipment_id} for order {order_id} (tracking: {tracking_number})") return shipment_id, tracking_number async def cancel_shipment(self, shipment_id: str) -> None: @@ -515,19 +511,19 @@ async def run_recovery_iteration( processed = 0 for saga_id in ids: try: - logger.info("Recovering saga %s...", saga_id) + logger.info(f"Recovering saga {saga_id}...") await recover_saga(saga, saga_id, context_builder, container, storage) - logger.info("Saga %s recovered successfully.", saga_id) + logger.info(f"Saga {saga_id} recovered successfully.") processed += 1 except RuntimeError as e: if "recovered in" in str(e) and "state" in str(e): - logger.info("Saga %s recovery completed compensation: %s", saga_id, e) + logger.info(f"Saga {saga_id} recovery completed compensation: {traceback.format_exc()}") processed += 1 else: - logger.exception("Saga %s recovery failed: %s", saga_id, e) + logger.exception(f"Saga {saga_id} recovery failed: {traceback.format_exc()}") processed += 1 - except Exception as e: - logger.exception("Saga %s recovery failed: %s", saga_id, e) + except Exception: + logger.exception(f"Saga {saga_id} recovery failed: {traceback.format_exc()}") processed += 1 return processed @@ -551,7 +547,7 @@ async def recovery_loop( iteration = 0 while True: iteration += 1 - logger.info("Recovery iteration %s", iteration) + logger.info(f"Recovery iteration {iteration}") try: processed = await run_recovery_iteration( storage, @@ -559,17 +555,17 @@ async def recovery_loop( OrderContext, ) if processed > 0: - logger.info("Processed %s saga(s) this iteration.", processed) + logger.info(f"Processed {processed} saga(s) this iteration.") else: logger.debug("No sagas to recover.") except asyncio.CancelledError: logger.info("Recovery loop cancelled.") raise except Exception as e: - logger.exception("Recovery iteration failed: %s", e) + logger.exception(f"Recovery iteration failed: {traceback.format_exc()}") if max_iterations is not None and iteration >= max_iterations: - logger.info("Reached max_iterations=%s, stopping.", max_iterations) + logger.info(f"Reached max_iterations={max_iterations}, stopping.") break await asyncio.sleep(interval_seconds) diff --git a/examples/saga_sqlalchemy_storage.py b/examples/saga_sqlalchemy_storage.py index d600026..e80e69e 100644 --- a/examples/saga_sqlalchemy_storage.py +++ b/examples/saga_sqlalchemy_storage.py @@ -140,13 +140,13 @@ async def setup_database(engine: AsyncEngine) -> None: async def main() -> None: - # 1. Create SQLAlchemy Engine with Connection Pool - # SQLAlchemy creates a pool by default (QueuePool for most dialects, SingletonThreadPool for SQLite) """ Run a demonstration that executes an OrderSaga using an async SQLAlchemy engine and persistent SqlAlchemySagaStorage. - + Initializes a pooled async SQLAlchemy engine and schema, creates a session factory and SqlAlchemySagaStorage, bootstraps a mediator with a DI container and saga mapper, runs an OrderSaga while streaming step results to stdout, and then reloads and prints the persisted saga state and step history before disposing the engine. """ + # 1. Create SQLAlchemy Engine with Connection Pool + # SQLAlchemy creates a pool by default (QueuePool for most dialects, SingletonThreadPool for SQLite) engine = create_async_engine( DB_URL, echo=False, # Set to True to see SQL queries diff --git a/src/cqrs/saga/compensation.py b/src/cqrs/saga/compensation.py index 53dd1ec..962197a 100644 --- a/src/cqrs/saga/compensation.py +++ b/src/cqrs/saga/compensation.py @@ -105,9 +105,6 @@ async def compensate_steps( "compensate", SagaStepStatus.COMPLETED, ) - if self._on_after_compensate_step is not None: - await self._on_after_compensate_step() - except Exception as compensation_error: await self._storage.log_step( self._saga_id, @@ -118,6 +115,19 @@ async def compensate_steps( ) # Store both step and error for better error reporting compensation_errors.append((step, compensation_error)) + continue + + # Callback only after successful step compensation; failures are not treated as step failure + if self._on_after_compensate_step is not None: + try: + await self._on_after_compensate_step() + except Exception as callback_error: + logger.error( + "on_after_compensate_step failed (e.g. run.commit): %s", + callback_error, + exc_info=callback_error, + ) + raise # If compensation failed after all retries if compensation_errors: diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index 2edd336..503f70c 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -86,6 +86,9 @@ def __init__( self._completed_steps: list[SagaStepHandler[ContextT, typing.Any]] = [] self._error: BaseException | None = None self._compensated: bool = False + self._comp_retry_count = compensation_retry_count + self._comp_retry_delay = compensation_retry_delay + self._comp_retry_backoff = compensation_retry_backoff self._saga_id = saga_id or uuid.uuid4() self._is_new_saga = saga_id is None @@ -112,9 +115,9 @@ def __init__( self._saga_id, context, storage, - compensation_retry_count, - compensation_retry_delay, - compensation_retry_backoff, + self._comp_retry_count, + self._comp_retry_delay, + self._comp_retry_backoff, ) @property @@ -172,6 +175,51 @@ async def __aiter__( async for step_result in self._execute(None): yield step_result + def _build_run_scoped_components( + self, + run: SagaStorageRun, + ) -> tuple[ + SagaStateManager, + SagaRecoveryManager, + SagaStepExecutor[ContextT], + FallbackStepExecutor[ContextT], + SagaCompensator[ContextT], + ]: + """Build state manager, recovery manager, executors, and compensator for a storage run (checkpoint commits).""" + state_manager = SagaStateManager(self._saga_id, run) + recovery_manager = SagaRecoveryManager( + self._saga_id, + run, + self._container, + self._saga.steps, + ) + step_executor = SagaStepExecutor( + self._context, + self._container, + state_manager, + ) + fallback_executor = FallbackStepExecutor( + self._context, + self._container, + state_manager, + ) + compensator = SagaCompensator( + self._saga_id, + self._context, + run, + self._comp_retry_count, + self._comp_retry_delay, + self._comp_retry_backoff, + on_after_compensate_step=run.commit, + ) + return ( + state_manager, + recovery_manager, + step_executor, + fallback_executor, + compensator, + ) + async def _execute( self, run: SagaStorageRun | None, @@ -189,32 +237,13 @@ async def _execute( RuntimeError: If the saga was recovered in COMPENSATING or FAILED state and compensation was completed, forward execution is not allowed. """ if run is not None: - state_manager = SagaStateManager(self._saga_id, run) - recovery_manager = SagaRecoveryManager( - self._saga_id, - run, - self._container, - self._saga.steps, - ) - step_executor = SagaStepExecutor( - self._context, - self._container, - state_manager, - ) - fallback_executor = FallbackStepExecutor( - self._context, - self._container, + ( state_manager, - ) - compensator = SagaCompensator( - self._saga_id, - self._context, - run, - self._compensator._retry_count, - self._compensator._retry_delay, - self._compensator._retry_backoff, - on_after_compensate_step=run.commit, - ) + recovery_manager, + step_executor, + fallback_executor, + compensator, + ) = self._build_run_scoped_components(run) else: state_manager = self._state_manager recovery_manager = self._recovery_manager diff --git a/src/cqrs/saga/storage/protocol.py b/src/cqrs/saga/storage/protocol.py index 701259c..870da87 100644 --- a/src/cqrs/saga/storage/protocol.py +++ b/src/cqrs/saga/storage/protocol.py @@ -19,7 +19,8 @@ async def create_saga( saga_id: uuid.UUID, name: str, context: dict[str, typing.Any], - ) -> None: """ + ) -> None: + """ Create a new saga execution record with initial PENDING status and version 1. Parameters: @@ -27,13 +28,14 @@ async def create_saga( name (str): Human-friendly name used for diagnostics and filtering. context (dict[str, Any]): JSON-serializable initial saga context to persist. """ - ... + async def update_context( self, saga_id: uuid.UUID, context: dict[str, typing.Any], current_version: int | None = None, - ) -> None: """ + ) -> None: + """ Persist a snapshot of the saga's execution context, optionally using optimistic locking. Parameters: @@ -45,12 +47,13 @@ async def update_context( Raises: SagaConcurrencyError: If `current_version` is provided and does not match the stored version. """ - ... + async def update_status( self, saga_id: uuid.UUID, status: SagaStatus, - ) -> None: """ + ) -> None: + """ Set the global status for the saga identified by `saga_id`. Parameters: @@ -60,7 +63,7 @@ async def update_status( Notes: This operation does not commit the storage session; the caller must call `commit()` on the active run or session to persist the change. """ - ... + async def log_step( self, saga_id: uuid.UUID, @@ -68,7 +71,8 @@ async def log_step( action: typing.Literal["act", "compensate"], status: SagaStepStatus, details: str | None = None, - ) -> None: """ + ) -> None: + """ Append a step transition to the saga's execution log. Parameters: @@ -78,13 +82,14 @@ async def log_step( status (SagaStepStatus): The step transition status to record (e.g., started, completed, failed, compensated). details (str | None): Optional human-readable details or diagnostics about the transition. """ - ... + async def load_saga_state( self, saga_id: uuid.UUID, *, read_for_update: bool = False, - ) -> tuple[SagaStatus, dict[str, typing.Any], int]: """ + ) -> tuple[SagaStatus, dict[str, typing.Any], int]: + """ Load the current saga execution state. Parameters: @@ -94,11 +99,12 @@ async def load_saga_state( Returns: tuple[SagaStatus, dict[str, Any], int]: A tuple containing the saga's global status, the latest persisted context (JSON-serializable), and the current optimistic-locking version number. """ - ... + async def get_step_history( self, saga_id: uuid.UUID, - ) -> list[SagaLogEntry]: """ + ) -> list[SagaLogEntry]: + """ Retrieve the chronological step log for a saga. Parameters: @@ -107,19 +113,20 @@ async def get_step_history( Returns: list[SagaLogEntry]: Ordered list of step log entries for the saga, from oldest to newest. """ - ... - async def commit(self) -> None: """ -Finalize the storage run by persisting and committing all pending changes made during this session. - -This method makes the run's checkpointed changes durable; the caller is responsible for invoking commit at logical checkpoints to persist session state. -""" -... - async def rollback(self) -> None: """ -Abort the current storage run and revert any uncommitted changes in the session. - -This releases the run's transactional state without persisting pending updates so that the storage remains as it was before the run began. -""" -... + + async def commit(self) -> None: + """ + Finalize the storage run by persisting and committing all pending changes made during this session. + + This method makes the run's checkpointed changes durable; the caller is responsible for invoking commit at logical checkpoints to persist session state. + """ + + async def rollback(self) -> None: + """ + Abort the current storage run and revert any uncommitted changes in the session. + + This releases the run's transactional state without persisting pending updates so that the storage remains as it was before the run began. + """ class ISagaStorage(abc.ABC): diff --git a/src/cqrs/saga/storage/sqlalchemy.py b/src/cqrs/saga/storage/sqlalchemy.py index b85c870..3607025 100644 --- a/src/cqrs/saga/storage/sqlalchemy.py +++ b/src/cqrs/saga/storage/sqlalchemy.py @@ -193,15 +193,10 @@ async def update_context( ) if current_version is not None: stmt = stmt.where(SagaExecutionModel.version == current_version) - stmt = stmt.values( - context=context, - version=SagaExecutionModel.version + 1, - ) - else: - stmt = stmt.values( - context=context, - version=SagaExecutionModel.version + 1, - ) + stmt = stmt.values( + context=context, + version=SagaExecutionModel.version + 1, + ) result = await self._session.execute(stmt) if result.rowcount == 0: # type: ignore[attr-defined] if current_version is not None: @@ -435,19 +430,12 @@ async def update_context( stmt = sqlalchemy.update(SagaExecutionModel).where( SagaExecutionModel.id == saga_id, ) - if current_version is not None: stmt = stmt.where(SagaExecutionModel.version == current_version) - stmt = stmt.values( - context=context, - version=SagaExecutionModel.version + 1, - ) - else: - # If no version check, just increment version - stmt = stmt.values( - context=context, - version=SagaExecutionModel.version + 1, - ) + stmt = stmt.values( + context=context, + version=SagaExecutionModel.version + 1, + ) result = await session.execute(stmt) diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py index e69de29..c038b0c 100644 --- a/tests/benchmarks/__init__.py +++ b/tests/benchmarks/__init__.py @@ -0,0 +1 @@ +# Benchmark package; shared helpers in storage_legacy.py diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index cee93f3..ca87358 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -5,7 +5,6 @@ """ import asyncio -import contextlib import dataclasses import typing @@ -16,7 +15,8 @@ from cqrs.saga.saga import Saga from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.memory import MemorySagaStorage -from cqrs.saga.storage.protocol import SagaStorageRun + +from ..storage_legacy import MemorySagaStorageLegacy @dataclasses.dataclass @@ -153,21 +153,6 @@ def memory_storage() -> MemorySagaStorage: return MemorySagaStorage() -class MemorySagaStorageLegacy(MemorySagaStorage): - """Memory storage without create_run: forces legacy path (commit per call).""" - - def create_run( - self, - ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """ - Indicate that creating a scoped run context is not supported for the legacy in-memory storage used in benchmarks. - - Raises: - NotImplementedError: Always raised with message "Legacy storage: create_run disabled for benchmark". - """ - raise NotImplementedError("Legacy storage: create_run disabled for benchmark") - - @pytest.fixture def memory_storage_legacy() -> MemorySagaStorageLegacy: """ @@ -180,21 +165,11 @@ def memory_storage_legacy() -> MemorySagaStorageLegacy: @pytest.fixture -def saga_with_memory_storage( - saga_container: SagaContainer, - memory_storage: MemorySagaStorage, -) -> Saga[OrderContext]: +def saga_with_memory_storage() -> Saga[OrderContext]: """ Create an OrderSaga configured with reserve-inventory, process-payment, and ship-order steps. - - This factory accepts the saga container and memory storage as fixture dependencies (they are not used by this function) and returns a Saga subclass instance with the three ordered step handlers: ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep. - - Parameters: - saga_container (SagaContainer): Fixture-provided container (unused). - memory_storage (MemorySagaStorage): Fixture-provided memory storage (unused). - - Returns: - Saga[OrderContext]: An OrderSaga instance wired with the predefined steps. + + Returns a Saga subclass instance with the three ordered step handlers: ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep. No fixture dependencies. """ class OrderSaga(Saga[OrderContext]): steps = [ReserveInventoryStep, ProcessPaymentStep, ShipOrderStep] @@ -261,7 +236,6 @@ def test_benchmark_saga_memory_run_ten_transactions( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, - memory_storage: MemorySagaStorage, ): """Benchmark 10 saga transactions in sequence, scoped run (memory storage).""" diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index 3036c10..9c445e4 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -4,15 +4,12 @@ - Benchmarks named *_legacy_* use the legacy path (no create_run, commit per storage call). """ -import contextlib - import pytest from sqlalchemy.ext.asyncio import async_sessionmaker from cqrs.saga.saga import Saga -from cqrs.saga.storage.protocol import SagaStorageRun -from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage +from ..storage_legacy import SqlAlchemySagaStorageLegacy from .test_benchmark_saga_memory import ( OrderContext, ProcessPaymentStep, @@ -22,21 +19,6 @@ ) -class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): - """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" - - def create_run( - self, - ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """ - Disable creation of a scoped SagaStorageRun for legacy storage used in benchmarks. - - Raises: - NotImplementedError: Always raised with message "Legacy storage: create_run disabled for benchmark". - """ - raise NotImplementedError("Legacy storage: create_run disabled for benchmark") - - @pytest.fixture def saga_container() -> SagaContainer: """ diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index 852c264..e3809f8 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -5,7 +5,6 @@ """ import asyncio -import contextlib import dataclasses import typing @@ -16,7 +15,8 @@ from cqrs.saga.saga import Saga from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.memory import MemorySagaStorage -from cqrs.saga.storage.protocol import SagaStorageRun + +from ..storage_legacy import MemorySagaStorageLegacy @dataclasses.dataclass @@ -150,24 +150,6 @@ def memory_storage() -> MemorySagaStorage: return MemorySagaStorage() -class MemorySagaStorageLegacy(MemorySagaStorage): - """Memory storage without create_run: forces legacy path (commit per call).""" - - def create_run( - self, - ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """ - Disable creation of scoped storage runs for the legacy storage variant used in benchmarks. - - Returns: - contextlib.AbstractAsyncContextManager[SagaStorageRun]: An async context manager yielding a `SagaStorageRun` (disabled in this legacy implementation). - - Raises: - NotImplementedError: Always raised with the message "Legacy storage: create_run disabled for benchmark". - """ - raise NotImplementedError("Legacy storage: create_run disabled for benchmark") - - @pytest.fixture def memory_storage_legacy() -> MemorySagaStorageLegacy: """ @@ -255,7 +237,6 @@ def test_benchmark_saga_memory_run_ten_transactions( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, - memory_storage: MemorySagaStorage, ): """Benchmark 10 saga transactions in sequence, scoped run (memory storage).""" diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index b494a53..c2be503 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -4,15 +4,12 @@ - Benchmarks named *_legacy_* use the legacy path (no create_run, commit per storage call). """ -import contextlib - import pytest from sqlalchemy.ext.asyncio import async_sessionmaker from cqrs.saga.saga import Saga -from cqrs.saga.storage.protocol import SagaStorageRun -from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage +from ..storage_legacy import SqlAlchemySagaStorageLegacy from .test_benchmark_saga_memory import ( OrderContext, ProcessPaymentStep, @@ -22,24 +19,6 @@ ) -class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): - """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" - - def create_run( - self, - ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """ - Disable scoped run creation for legacy storage used in benchmarks. - - This storage intentionally does not provide a scoped `create_run` context manager. - Calling this method raises a NotImplementedError to indicate the legacy path is in use. - - Raises: - NotImplementedError: Always raised to indicate scoped run creation is disabled for legacy storage. - """ - raise NotImplementedError("Legacy storage: create_run disabled for benchmark") - - @pytest.fixture def saga_container() -> SagaContainer: """ diff --git a/tests/benchmarks/storage_legacy.py b/tests/benchmarks/storage_legacy.py new file mode 100644 index 0000000..2da7915 --- /dev/null +++ b/tests/benchmarks/storage_legacy.py @@ -0,0 +1,29 @@ +"""Shared legacy storage classes for benchmark tests (no create_run; commit-per-call path).""" + +from __future__ import annotations + +import contextlib + +from cqrs.saga.storage.memory import MemorySagaStorage +from cqrs.saga.storage.protocol import SagaStorageRun +from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage + + +class MemorySagaStorageLegacy(MemorySagaStorage): + """Memory storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """Raise NotImplementedError so benchmarks use the legacy commit-per-call path.""" + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") + + +class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): + """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """Raise NotImplementedError so benchmarks use the legacy commit-per-call path.""" + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") diff --git a/tests/unit/test_saga/test_saga_basic.py b/tests/unit/test_saga/test_saga_basic.py index d06df57..b5a9439 100644 --- a/tests/unit/test_saga/test_saga_basic.py +++ b/tests/unit/test_saga/test_saga_basic.py @@ -313,4 +313,4 @@ async def test_saga_step_result_contains_correct_metadata( assert step_result.error_message is None assert step_result.error_traceback is None assert step_result.error_type is None - assert step_result.saga_id is not None + assert step_result.saga_id is not None \ No newline at end of file From 2738cbc19181e1c10c0c11dc82a62935c220afe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 13:50:48 +0300 Subject: [PATCH 06/17] Fix deadlocks --- .github/workflows/codspeed.yml | 13 + .github/workflows/tests.yml | 18 + docker-compose-dev.yml | 10 + docker-compose-test.yml | 10 + examples/saga_recovery_scheduler.py | 6 +- examples/saga_sqlalchemy_storage.py | 2 +- pyproject.toml | 1 + src/cqrs/saga/compensation.py | 10 +- src/cqrs/saga/saga.py | 20 +- src/cqrs/saga/storage/protocol.py | 30 +- src/cqrs/saga/storage/sqlalchemy.py | 47 ++- .../dataclasses/test_benchmark_saga_memory.py | 17 +- .../test_benchmark_saga_sqlalchemy.py | 15 +- .../default/test_benchmark_saga_memory.py | 19 +- .../default/test_benchmark_saga_sqlalchemy.py | 11 +- tests/integration/fixtures.py | 80 ++-- ...=> test_saga_mediator_sqlalchemy_mysql.py} | 152 ++----- .../test_saga_mediator_sqlalchemy_postgres.py | 338 +++++++++++++++ ... => test_saga_storage_sqlalchemy_mysql.py} | 265 ++---------- .../test_saga_storage_sqlalchemy_postgres.py | 393 ++++++++++++++++++ tests/pytest-config.ini | 2 + tests/unit/test_saga/test_saga_basic.py | 2 +- 22 files changed, 1012 insertions(+), 449 deletions(-) rename tests/integration/{test_saga_mediator_sqlalchemy.py => test_saga_mediator_sqlalchemy_mysql.py} (76%) create mode 100644 tests/integration/test_saga_mediator_sqlalchemy_postgres.py rename tests/integration/{test_saga_storage_sqlalchemy.py => test_saga_storage_sqlalchemy_mysql.py} (63%) create mode 100644 tests/integration/test_saga_storage_sqlalchemy_postgres.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index ca23eed..df0f886 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -46,6 +46,19 @@ jobs: echo "MySQL did not become ready in time" exit 1 + - name: Wait for PostgreSQL + run: | + for i in $(seq 1 30); do + if docker compose -f docker-compose-test.yml exec -T postgres_tests pg_isready -h localhost -U cqrs -q 2>/dev/null; then + echo "PostgreSQL is ready" + exit 0 + fi + echo "Waiting for PostgreSQL... ($i/30)" + sleep 2 + done + echo "PostgreSQL did not become ready in time" + exit 1 + - name: Wait for Redis run: | for i in $(seq 1 15); do diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12f31db..3f69bfc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,6 +69,7 @@ jobs: vermin --target=3.10- --violations --eval-annotations --backport typing_extensions --exclude=venv --exclude=build --exclude=.git --exclude=.venv src examples tests test: + name: test (py ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -105,6 +106,19 @@ jobs: echo "MySQL did not become ready in time" exit 1 + - name: Wait for PostgreSQL + run: | + for i in $(seq 1 30); do + if docker compose -f docker-compose-test.yml exec -T postgres_tests pg_isready -h localhost -U cqrs -q 2>/dev/null; then + echo "PostgreSQL is ready" + exit 0 + fi + echo "Waiting for PostgreSQL... ($i/30)" + sleep 2 + done + echo "PostgreSQL did not become ready in time" + exit 1 + - name: Wait for Redis run: | for i in $(seq 1 15); do @@ -119,6 +133,10 @@ jobs: exit 1 - name: Run all tests with coverage + env: + DATABASE_DSN: mysql+asyncmy://cqrs:cqrs@localhost:3307/test_cqrs + DATABASE_DSN_MYSQL: mysql+asyncmy://cqrs:cqrs@localhost:3307/test_cqrs + DATABASE_DSN_POSTGRESQL: postgresql+asyncpg://cqrs:cqrs@localhost:5433/cqrs run: | pytest -c ./tests/pytest-config.ini --cov=src --cov-report=xml --cov-report=term -o cache_dir=/tmp/pytest_cache ./tests/unit ./tests/integration diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c8d255a..064b5b1 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -15,6 +15,16 @@ services: command: --init-file /data/application/init.sql volumes: - ./tests/init_database.sql:/data/application/init.sql + postgres_dev: + image: postgres:latest + hostname: postgres-dev + restart: always + environment: + POSTGRES_USER: cqrs + POSTGRES_PASSWORD: cqrs + POSTGRES_DB: cqrs + ports: + - "5433:5432" kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 18b377f..4e97ab1 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -15,6 +15,16 @@ services: command: --init-file /data/application/init.sql volumes: - ./tests/init_database.sql:/data/application/init.sql + postgres_tests: + image: postgres:latest + hostname: postgres-test + restart: always + environment: + POSTGRES_USER: cqrs + POSTGRES_PASSWORD: cqrs + POSTGRES_DB: cqrs + ports: + - "5433:5432" redis_tests: image: redis:7.2 hostname: redis diff --git a/examples/saga_recovery_scheduler.py b/examples/saga_recovery_scheduler.py index 387ac13..d838fe5 100644 --- a/examples/saga_recovery_scheduler.py +++ b/examples/saga_recovery_scheduler.py @@ -561,7 +561,7 @@ async def recovery_loop( except asyncio.CancelledError: logger.info("Recovery loop cancelled.") raise - except Exception as e: + except Exception: logger.exception(f"Recovery iteration failed: {traceback.format_exc()}") if max_iterations is not None and iteration >= max_iterations: @@ -618,7 +618,7 @@ async def create_interrupted_saga(storage: MemorySagaStorage) -> uuid.UUID: async def main() -> None: """ Run the saga recovery scheduler demo and display its outcome. - + Sets up an in-memory saga storage, creates a simulated interrupted saga and marks it stale, runs the recovery loop for three iterations (using the module's recovery_loop and recovery configuration constants), then loads and prints the final saga state. """ print("\n" + "=" * 70) @@ -660,4 +660,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/examples/saga_sqlalchemy_storage.py b/examples/saga_sqlalchemy_storage.py index e80e69e..3b14b46 100644 --- a/examples/saga_sqlalchemy_storage.py +++ b/examples/saga_sqlalchemy_storage.py @@ -207,4 +207,4 @@ def saga_mapper(mapper: cqrs.SagaMap) -> None: if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 3ff60e0..fc0491a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ dev = [ "pytest-env==0.6.2", "cryptography==42.0.2", "asyncmy==0.2.9", + "asyncpg>=0.29.0", "redis>=5.0.0", # Circuit breaker for tests "aiobreaker>=0.3.0" # from aiobreaker diff --git a/src/cqrs/saga/compensation.py b/src/cqrs/saga/compensation.py index 962197a..699ee82 100644 --- a/src/cqrs/saga/compensation.py +++ b/src/cqrs/saga/compensation.py @@ -27,7 +27,7 @@ def __init__( ) -> None: """ Create a SagaCompensator configured to perform compensation of completed saga steps with retry and optional post-step callback. - + Parameters: saga_id: Identifier of the saga. context: Saga execution context passed to step compensation handlers. @@ -51,12 +51,12 @@ async def compensate_steps( ) -> None: """ Compensates completed saga steps in reverse order, applying retry logic and recording step statuses. - + Compensates each handler from last to first, skipping steps already recorded as compensated in the saga history. Updates the saga status to COMPENSATING at the start and logs per-step statuses (STARTED, COMPLETED, FAILED) in storage. After a step completes, the optional on_after_compensate_step callback (if provided) is awaited. If any step fails after all retry attempts, the saga is marked as FAILED. If no completed steps are provided, no compensation is attempted and the saga is marked as FAILED. - + Parameters: completed_steps (list[SagaStepHandler[ContextT, typing.Any]]): Handlers corresponding to steps that completed during the saga; these will be compensated in reverse order. - + Returns: None """ @@ -184,4 +184,4 @@ async def _compensate_step_with_retry( # If we get here, all retries failed if last_exception: - raise last_exception \ No newline at end of file + raise last_exception diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index 503f70c..db0709a 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -157,10 +157,10 @@ async def __aiter__( ) -> typing.AsyncIterator[SagaStepResult[ContextT, typing.Any]]: """ Execute saga steps sequentially and yield each step result. - + Implements the Strict Backward Recovery strategy: if the saga is in COMPENSATING or FAILED status, forward execution is never resumed. When the underlying storage provides create_run(), execution is performed within a per-saga run with checkpoint commits; otherwise the legacy run-less path is used. Returns: - AsyncIterator[SagaStepResult[ContextT, typing.Any]]: An async iterator that yields the result for each executed saga step in order. + AsyncIterator[SagaStepResult[ContextT, typing.Any]]: An async iterator that yields the result for each executed saga step in order. """ try: run_cm = self._storage.create_run() @@ -226,13 +226,13 @@ async def _execute( ) -> typing.AsyncIterator[SagaStepResult[ContextT, typing.Any]]: """ Execute the saga's configured steps, using the provided storage run for checkpointed operations when available, and perform recovery and compensation as required. - + Parameters: run (SagaStorageRun | None): Optional per-saga storage run. When provided, the run is used for loading saga state, creating run-scoped managers/executors, and committing at checkpoint boundaries. When None, the transaction's internal managers and executors are used. - + Returns: Async iterator that yields SagaStepResult values for each step that completes; each yielded result will include the transaction's saga_id. - + Raises: RuntimeError: If the saga was recovered in COMPENSATING or FAILED state and compensation was completed, forward execution is not allowed. """ @@ -258,6 +258,8 @@ async def _execute( self._saga.__class__.__name__, self._context, ) + if run is not None: + await run.commit() await state_manager.update_status(SagaStatus.RUNNING) if run is not None: await run.commit() @@ -308,10 +310,14 @@ async def _execute( completed_step_names = await recovery_manager.load_completed_step_names() except ValueError: + if run is not None: + await run.rollback() await state_manager.create_saga( self._saga.__class__.__name__, self._context, ) + if run is not None: + await run.commit() await state_manager.update_status(SagaStatus.RUNNING) if run is not None: await run.commit() @@ -390,7 +396,7 @@ async def _execute( async def _compensate(self) -> None: """ Mark the transaction as compensated and run compensation for all completed steps in reverse order. - + Sets an internal flag to prevent repeated compensation and delegates to the compensator which applies the configured retry behavior. """ # Prevent double compensation @@ -519,4 +525,4 @@ def transaction( compensation_retry_count=compensation_retry_count, compensation_retry_delay=compensation_retry_delay, compensation_retry_backoff=compensation_retry_backoff, - ) \ No newline at end of file + ) diff --git a/src/cqrs/saga/storage/protocol.py b/src/cqrs/saga/storage/protocol.py index 870da87..27fcd88 100644 --- a/src/cqrs/saga/storage/protocol.py +++ b/src/cqrs/saga/storage/protocol.py @@ -22,7 +22,7 @@ async def create_saga( ) -> None: """ Create a new saga execution record with initial PENDING status and version 1. - + Parameters: saga_id (uuid.UUID): Unique identifier for the saga (primary key). name (str): Human-friendly name used for diagnostics and filtering. @@ -37,13 +37,13 @@ async def update_context( ) -> None: """ Persist a snapshot of the saga's execution context, optionally using optimistic locking. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to update. context (dict[str, Any]): JSON-serializable context object to store as the new snapshot. current_version (int | None): If provided, perform an optimistic-locking update that succeeds only if the stored version matches this value; on success the stored version is incremented. - + Raises: SagaConcurrencyError: If `current_version` is provided and does not match the stored version. """ @@ -55,11 +55,11 @@ async def update_status( ) -> None: """ Set the global status for the saga identified by `saga_id`. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to update. status (SagaStatus): New global status to persist (for example RUNNING, COMPLETED, COMPENSATING). - + Notes: This operation does not commit the storage session; the caller must call `commit()` on the active run or session to persist the change. """ @@ -74,7 +74,7 @@ async def log_step( ) -> None: """ Append a step transition to the saga's execution log. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose log will be appended. step_name (str): Logical name of the step (used for diagnostics and replay). @@ -91,14 +91,15 @@ async def load_saga_state( ) -> tuple[SagaStatus, dict[str, typing.Any], int]: """ Load the current saga execution state. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to load. read_for_update (bool): If True, acquire a database lock for update to prevent concurrent modifications. - + Returns: tuple[SagaStatus, dict[str, Any], int]: A tuple containing the saga's global status, the latest persisted context (JSON-serializable), and the current optimistic-locking version number. """ + ... async def get_step_history( self, @@ -106,13 +107,14 @@ async def get_step_history( ) -> list[SagaLogEntry]: """ Retrieve the chronological step log for a saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose step history to retrieve. - + Returns: list[SagaLogEntry]: Ordered list of step log entries for the saga, from oldest to newest. """ + ... async def commit(self) -> None: """ @@ -343,13 +345,13 @@ def create_run( ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: """ Create a scoped async run context for a single saga execution session with checkpointed commits. - + The context manager yields a SagaStorageRun that provides the same mutation/read methods as the storage but does not commit automatically; the caller must call commit() or rollback() at desired checkpoints. - + Returns: contextlib.AbstractAsyncContextManager[SagaStorageRun]: Async context manager yielding a SagaStorageRun session. - + Raises: NotImplementedError: If the storage backend does not support scoped runs. """ - raise NotImplementedError("This storage does not support create_run()") \ No newline at end of file + raise NotImplementedError("This storage does not support create_run()") diff --git a/src/cqrs/saga/storage/sqlalchemy.py b/src/cqrs/saga/storage/sqlalchemy.py index 3607025..f892757 100644 --- a/src/cqrs/saga/storage/sqlalchemy.py +++ b/src/cqrs/saga/storage/sqlalchemy.py @@ -139,7 +139,7 @@ class _SqlAlchemySagaStorageRun(SagaStorageRun): def __init__(self, session: AsyncSession) -> None: """ Initialize the run wrapper with an async SQLAlchemy session. - + Parameters: session (AsyncSession): The AsyncSession instance scoped to this run, used for all database operations. """ @@ -153,9 +153,9 @@ async def create_saga( ) -> None: """ Create and stage a new saga execution record in the current session with initial metadata. - + Creates a SagaExecutionModel for the given saga identifier with status set to PENDING, version set to 1, and recovery_attempts set to 0, and adds it to the active session without committing the transaction. - + Parameters: saga_id (uuid.UUID): Unique identifier for the saga execution. name (str): Human-readable name of the saga. @@ -179,12 +179,12 @@ async def update_context( ) -> None: """ Update the stored context for a saga and increment its version, optionally enforcing an optimistic version check. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to update. context (dict[str, typing.Any]): New serialized saga context to persist. current_version (int | None): If provided, require the saga's current version to match this value before updating. - + Raises: SagaConcurrencyError: If an optimistic version check fails (indicating a concurrent modification) or if the saga does not exist when a version was supplied. """ @@ -219,11 +219,11 @@ async def update_status( ) -> None: """ Update the stored status of a saga execution and increment its optimistic-lock version. - + Parameters: saga_id (uuid.UUID): Identifier of the saga execution to update. status (SagaStatus): New status to set for the saga. - + Note: The update is executed in the active database session; a commit is required to persist the change. """ @@ -246,7 +246,7 @@ async def log_step( ) -> None: """ Record a saga step event by creating and staging a log entry in the active session. - + Parameters: saga_id (uuid.UUID): Identifier of the saga execution. step_name (str): Name of the step being recorded. @@ -271,14 +271,14 @@ async def load_saga_state( ) -> tuple[SagaStatus, dict[str, typing.Any], int]: """ Load the current execution state for a saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to load. read_for_update (bool): If true, acquire a row-level lock for update. - + Returns: tuple[SagaStatus, dict[str, Any], int]: The saga's status, its context dictionary, and the current version. - + Raises: ValueError: If no saga with the given id exists. """ @@ -305,10 +305,10 @@ async def get_step_history( ) -> list[SagaLogEntry]: """ Retrieve chronological step log entries for the given saga. - + Parameters: saga_id (uuid.UUID): UUID of the saga whose step history to fetch. - + Returns: list[SagaLogEntry]: List of log entries ordered by creation time. Each entry's `timestamp` is normalized to UTC if not already timezone-aware. @@ -343,7 +343,7 @@ async def commit(self) -> None: async def rollback(self) -> None: """ Revert all staged changes in the current session's transaction. - + This aborts the in-progress transaction associated with the run's AsyncSession, discarding any pending writes or flushes. """ @@ -354,7 +354,7 @@ class SqlAlchemySagaStorage(ISagaStorage): def __init__(self, session_factory: async_sessionmaker[AsyncSession]): """ Initialize the SQLAlchemy-based saga storage with a factory for creating async sessions. - + Parameters: session_factory (async_sessionmaker[AsyncSession]): Factory that produces new AsyncSession instances used for each storage run and operation. """ @@ -365,12 +365,13 @@ def create_run( ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: """ Create a scoped run that yields a SagaStorageRun bound to a fresh session. - + The returned context manager provides a run object whose lifecycle is tied to a single session. If an exception is raised inside the context, the run's transaction is rolled back; the session is always closed on exit. - + Returns: A context manager that yields a `SagaStorageRun`. On exception within the context, the run's `rollback()` is invoked and the session is closed when the context exits. """ + @contextlib.asynccontextmanager async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: async with self.session_factory() as session: @@ -391,15 +392,15 @@ async def create_saga( ) -> None: """ Create and persist a new saga execution record with initial metadata. - + Creates a SagaExecutionModel for the given saga_id and name, sets status to PENDING, version to 1, and recovery_attempts to 0, and commits it to the database. - + Parameters: saga_id (uuid.UUID): Unique identifier for the saga execution. name (str): Human-readable saga name. context (dict[str, typing.Any]): Initial saga context to store. - + Raises: SQLAlchemyError: If the database operation fails; the transaction is rolled back before the exception is propagated. """ @@ -604,11 +605,11 @@ async def increment_recovery_attempts( ) -> None: """ Increment the recovery attempts counter for the given saga execution and optionally update its status. - + Parameters: saga_id (uuid.UUID): Identifier of the saga execution to update. new_status (SagaStatus | None): If provided, set the saga's status to this value. - + Raises: ValueError: If no saga execution exists with the given `saga_id`. SQLAlchemyError: On database errors; the transaction is rolled back and the error is propagated. @@ -651,4 +652,4 @@ async def set_recovery_attempts( await session.commit() except SQLAlchemyError: await session.rollback() - raise \ No newline at end of file + raise diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index ca87358..a92f399 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -146,7 +146,7 @@ def saga_container() -> SagaContainer: def memory_storage() -> MemorySagaStorage: """ Provide a fresh in-memory saga storage instance for tests and benchmarks. - + Returns: MemorySagaStorage: A new MemorySagaStorage instance. """ @@ -157,7 +157,7 @@ def memory_storage() -> MemorySagaStorage: def memory_storage_legacy() -> MemorySagaStorageLegacy: """ Provide a legacy in-memory saga storage that does not support scoped runs. - + Returns: MemorySagaStorageLegacy: An in-memory saga storage whose `create_run` is disabled (raises `NotImplementedError`) for legacy-path benchmarks. """ @@ -171,6 +171,7 @@ def saga_with_memory_storage() -> Saga[OrderContext]: Returns a Saga subclass instance with the three ordered step handlers: ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep. No fixture dependencies. """ + class OrderSaga(Saga[OrderContext]): steps = [ReserveInventoryStep, ProcessPaymentStep, ShipOrderStep] @@ -189,7 +190,7 @@ def test_benchmark_saga_memory_full_transaction( async def run() -> None: """ Execute a full saga transaction using the module's memory-backed saga, advancing through every step. - + Creates an OrderContext with order_id "ord_1", user_id "user_1", and amount 100.0, opens a transaction using the provided saga container and memory storage, and iterates the transaction to exercise each step in the run path. """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -242,7 +243,7 @@ def test_benchmark_saga_memory_run_ten_transactions( async def run() -> None: """ Execute 10 sequential saga transactions using a fresh in-memory storage and context for each iteration. - + Each iteration creates a new MemorySagaStorage and OrderContext, opens a saga transaction with the provided container and storage, and iterates through the transaction steps without performing additional work. """ for i in range(10): @@ -278,7 +279,7 @@ def test_benchmark_saga_memory_legacy_full_transaction( async def run() -> None: """ Run a full-order saga transaction against the legacy in-memory storage used for benchmarks. - + Creates an OrderContext and executes the saga transaction using the provided saga container and legacy memory storage, iterating the transaction to completion. """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -310,7 +311,7 @@ class SingleStepSaga(Saga[OrderContext]): async def run() -> None: """ Run a saga transaction using the legacy memory storage and iterate its steps. - + Enters a transaction for an OrderContext (order_id "ord_1") with the registered saga_container and memory_storage_legacy, then iterates through the transaction steps without performing work for each step. """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -336,7 +337,7 @@ def test_benchmark_saga_memory_legacy_ten_transactions( async def run() -> None: """ Execute ten sequential saga transactions using MemorySagaStorageLegacy. - + Each iteration creates a new MemorySagaStorageLegacy and an OrderContext (with distinct order_id, user_id, and amount) and runs the configured saga transaction to completion by iterating through its steps. """ for i in range(10): @@ -354,4 +355,4 @@ async def run() -> None: async for _ in transaction: pass - benchmark(lambda: asyncio.run(run())) \ No newline at end of file + benchmark(lambda: asyncio.run(run())) diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index 9c445e4..615519e 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -8,6 +8,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker from cqrs.saga.saga import Saga +from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from ..storage_legacy import SqlAlchemySagaStorageLegacy from .test_benchmark_saga_memory import ( @@ -23,7 +24,7 @@ def saga_container() -> SagaContainer: """ Create and return a SagaContainer pre-registered with the ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep instances. - + Returns: SagaContainer: A container with the three steps already registered. """ @@ -99,7 +100,7 @@ class SingleStepSaga(Saga[OrderContext]): async def run_transaction() -> None: """ Execute the saga transaction lifecycle by entering the saga's transaction context and iterating its steps to completion. - + This function opens the saga transaction using the surrounding `saga`, `saga_container`, `storage`, and `context`, then consumes the transaction iterator to drive all saga steps to completion. """ async with saga.transaction( @@ -125,7 +126,7 @@ def test_benchmark_saga_sqlalchemy_legacy_full_transaction( ): """ Benchmark a full saga transaction using SQLAlchemy storage in legacy mode. - + Runs a complete saga (three-step OrderSaga) against SqlAlchemySagaStorageLegacy, which disables `create_run` so the storage exercises the legacy commit-per-call path. The benchmark executes the saga transaction in the provided event loop and database engine fixture. """ loop, engine = saga_benchmark_loop_and_engine @@ -142,7 +143,7 @@ def test_benchmark_saga_sqlalchemy_legacy_full_transaction( async def run_transaction() -> None: """ Execute the configured saga transaction and iterate through all its steps to completion. - + This coroutine opens a transaction using the surrounding `saga_sqlalchemy`, `saga_container`, `context`, and `storage` variables and consumes the transaction iterator without performing additional actions. """ async with saga_sqlalchemy.transaction( @@ -164,7 +165,7 @@ def test_benchmark_saga_sqlalchemy_legacy_single_step( ): """ Benchmark executing a single-step Saga using legacy SQLAlchemy storage (commit-per-call path). - + Constructs a SingleStepSaga with ReserveInventoryStep, creates a SqlAlchemySagaStorageLegacy backed by the provided engine/session factory, and measures running a full saga transaction (iterating the transaction to completion) using the provided event loop via the benchmark fixture. """ loop, engine = saga_benchmark_loop_and_engine @@ -186,7 +187,7 @@ class SingleStepSaga(Saga[OrderContext]): async def run_transaction() -> None: """ Execute the saga transaction lifecycle by entering the saga's transaction context and iterating its steps to completion. - + This function opens the saga transaction using the surrounding `saga`, `saga_container`, `storage`, and `context`, then consumes the transaction iterator to drive all saga steps to completion. """ async with saga.transaction( @@ -197,4 +198,4 @@ async def run_transaction() -> None: async for _ in transaction: pass - benchmark(lambda: loop.run_until_complete(run_transaction())) \ No newline at end of file + benchmark(lambda: loop.run_until_complete(run_transaction())) diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index e3809f8..e3aaefa 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -143,7 +143,7 @@ def saga_container() -> SagaContainer: def memory_storage() -> MemorySagaStorage: """ Create a fresh in-memory saga storage instance for tests. - + Returns: MemorySagaStorage: A new MemorySagaStorage used to persist saga state in memory. """ @@ -154,7 +154,7 @@ def memory_storage() -> MemorySagaStorage: def memory_storage_legacy() -> MemorySagaStorageLegacy: """ Create a MemorySagaStorageLegacy instance for legacy-path benchmarks. - + Returns: MemorySagaStorageLegacy: A storage instance where `create_run()` is disabled and will raise NotImplementedError if called. """ @@ -168,10 +168,11 @@ def saga_with_memory_storage( ) -> Saga[OrderContext]: """ Create an OrderSaga preconfigured with inventory reservation, payment processing, and shipping steps. - + Returns: Saga[OrderContext]: An instance configured with ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep. """ + class OrderSaga(Saga[OrderContext]): steps = [ReserveInventoryStep, ProcessPaymentStep, ShipOrderStep] @@ -190,7 +191,7 @@ def test_benchmark_saga_memory_full_transaction( async def run() -> None: """ Execute a full three-step OrderSaga transaction using the memory storage scoped-run path. - + Creates an OrderContext and runs the saga transaction to completion with the provided saga container and memory storage. """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -243,7 +244,7 @@ def test_benchmark_saga_memory_run_ten_transactions( async def run() -> None: """ Run ten sequential saga transactions, each using a new MemorySagaStorage and an OrderContext. - + Each iteration (i from 0 to 9) creates: - a fresh MemorySagaStorage, - an OrderContext with order_id "ord_i", user_id "user_i", and amount 100.0 + i, @@ -282,7 +283,7 @@ def test_benchmark_saga_memory_legacy_full_transaction( async def run() -> None: """ Execute a full OrderSaga transaction using the legacy memory storage path. - + Builds an OrderContext (order_id "ord_1", user_id "user_1", amount 100.0) and runs the saga_with_memory_storage transaction with saga_container and memory_storage_legacy, iterating the transaction to completion. """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -314,7 +315,7 @@ class SingleStepSaga(Saga[OrderContext]): async def run() -> None: """ Runs a full OrderSaga transaction using the legacy memory storage path. - + This coroutine executes the saga with an OrderContext and the MemorySagaStorageLegacy instance so the saga proceeds through all steps while exercising the legacy storage behavior (create_run disabled, commit-per-call path). """ context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) @@ -340,7 +341,7 @@ def test_benchmark_saga_memory_legacy_ten_transactions( async def run() -> None: """ Run ten sequential saga transactions using the legacy memory storage path. - + For each iteration this function creates a new MemorySagaStorageLegacy, constructs an OrderContext with a unique order_id and user_id and increasing amount, opens a saga transaction using the shared saga_with_memory_storage and saga_container, and iterates the transaction to completion. """ for i in range(10): @@ -358,4 +359,4 @@ async def run() -> None: async for _ in transaction: pass - benchmark(lambda: asyncio.run(run())) \ No newline at end of file + benchmark(lambda: asyncio.run(run())) diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index c2be503..34339c0 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -8,6 +8,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker from cqrs.saga.saga import Saga +from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from ..storage_legacy import SqlAlchemySagaStorageLegacy from .test_benchmark_saga_memory import ( @@ -23,7 +24,7 @@ def saga_container() -> SagaContainer: """ Create a SagaContainer pre-registered with the standard order saga steps. - + Returns: SagaContainer: Container with ReserveInventoryStep, ProcessPaymentStep, and ShipOrderStep registered. """ @@ -99,7 +100,7 @@ class SingleStepSaga(Saga[OrderContext]): async def run_transaction() -> None: """ Run the saga transaction to completion by iterating over its yielded steps using the configured context, container, and storage. - + This function is used by benchmarks to execute a full saga flow without performing additional work per step. """ async with saga.transaction( @@ -138,7 +139,7 @@ def test_benchmark_saga_sqlalchemy_legacy_full_transaction( async def run_transaction() -> None: """ Execute the saga transaction and iterate through all produced steps without performing any operations. - + Opens the saga's transaction context with the configured container and storage, then consumes every yielded step (no-op per step). Intended for benchmarking the transaction iteration path. """ async with saga_sqlalchemy.transaction( @@ -178,7 +179,7 @@ class SingleStepSaga(Saga[OrderContext]): async def run_transaction() -> None: """ Run the saga transaction to completion by iterating over its yielded steps using the configured context, container, and storage. - + This function is used by benchmarks to execute a full saga flow without performing additional work per step. """ async with saga.transaction( @@ -189,4 +190,4 @@ async def run_transaction() -> None: async for _ in transaction: pass - benchmark(lambda: loop.run_until_complete(run_transaction())) \ No newline at end of file + benchmark(lambda: loop.run_until_complete(run_transaction())) diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index 8965e73..cbb1f5b 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -11,6 +11,10 @@ dotenv.load_dotenv() DATABASE_DSN = os.environ.get("DATABASE_DSN", "") +# DSN для тестов саги: отдельные переменные для MySQL и PostgreSQL (задаются в pytest-config.ini / env). +DATABASE_DSN_MYSQL = os.environ.get("DATABASE_DSN_MYSQL", "") +DATABASE_DSN_POSTGRESQL = os.environ.get("DATABASE_DSN_POSTGRESQL", "") + @pytest.fixture(scope="function") async def init_orm(): @@ -40,55 +44,69 @@ async def session(init_orm): yield session -# Saga storage fixtures +# --- Saga storage: MySQL (отдельные фикстуры, поднимают схему и всё необходимое) --- + + @pytest.fixture(scope="session") -async def init_saga_orm(): - """Initialize saga storage tables - drops and creates tables BEFORE test only.""" +async def init_saga_orm_mysql(): + """Поднять схему саги для MySQL (DATABASE_DSN_MYSQL).""" from cqrs.saga.storage.sqlalchemy import Base + if not DATABASE_DSN_MYSQL: + pytest.skip("DATABASE_DSN_MYSQL not set") engine = create_async_engine( - DATABASE_DSN, + DATABASE_DSN_MYSQL, pool_pre_ping=True, pool_size=10, max_overflow=30, echo=False, ) - # Drop and create tables BEFORE test (not after) - # Use begin() to ensure tables are created, but don't keep transaction open async with engine.begin() as connect: await connect.run_sync(Base.metadata.drop_all) await connect.run_sync(Base.metadata.create_all) - - # Yield engine so it can be used for sessions - # Data will persist after test because we don't drop tables in cleanup yield engine - - # Cleanup: dispose engine but DON'T drop tables - keep data in DB await engine.dispose() @pytest.fixture(scope="session") -def saga_session_factory(init_saga_orm): - """Create a session factory for saga storage tests.""" - engine = init_saga_orm - return async_sessionmaker(engine, expire_on_commit=False, autocommit=False) +def saga_session_factory_mysql(init_saga_orm_mysql): + """Session factory для тестов саги на MySQL.""" + return async_sessionmaker( + init_saga_orm_mysql, + expire_on_commit=False, + autocommit=False, + ) + + +# --- Saga storage: PostgreSQL (отдельные фикстуры, поднимают схему и всё необходимое) --- @pytest.fixture(scope="session") -async def saga_session(saga_session_factory): - """Create a session for saga storage tests - commits data to persist.""" - # Use autocommit=False but ensure we commit explicitly - session = saga_session_factory() +async def init_saga_orm_postgres(): + """Поднять схему саги для PostgreSQL (DATABASE_DSN_POSTGRESQL).""" + from cqrs.saga.storage.sqlalchemy import Base - async with contextlib.aclosing(session): - try: - yield session - # Final commit before closing to ensure data persists - if session.in_transaction(): - await session.commit() - except Exception: - # Only rollback on exception - if session.in_transaction(): - await session.rollback() - raise - # No cleanup that would delete data - data persists in DB + if not DATABASE_DSN_POSTGRESQL: + pytest.skip("DATABASE_DSN_POSTGRESQL not set") + engine = create_async_engine( + DATABASE_DSN_POSTGRESQL, + pool_pre_ping=True, + pool_size=10, + max_overflow=30, + echo=False, + ) + async with engine.begin() as connect: + await connect.run_sync(Base.metadata.drop_all) + await connect.run_sync(Base.metadata.create_all) + yield engine + await engine.dispose() + + +@pytest.fixture(scope="session") +def saga_session_factory_postgres(init_saga_orm_postgres): + """Session factory для тестов саги на PostgreSQL.""" + return async_sessionmaker( + init_saga_orm_postgres, + expire_on_commit=False, + autocommit=False, + ) diff --git a/tests/integration/test_saga_mediator_sqlalchemy.py b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py similarity index 76% rename from tests/integration/test_saga_mediator_sqlalchemy.py rename to tests/integration/test_saga_mediator_sqlalchemy_mysql.py index 6326f59..dd42fc2 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py @@ -1,4 +1,4 @@ -"""Integration tests for SagaMediator with SqlAlchemySagaStorage.""" +"""Integration tests for SagaMediator with SqlAlchemySagaStorage (MySQL).""" import typing import uuid @@ -13,7 +13,6 @@ from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage -# Import test models from memory test file from tests.integration.test_saga_mediator_memory import ( FailingOrderSaga, FailingStep, @@ -34,7 +33,6 @@ ) -# Container setup (reuse from memory test) class _TestContainer: """Test container that resolves step handlers, sagas, and event handlers.""" @@ -52,7 +50,6 @@ def __init__(self, storage: SqlAlchemySagaStorage) -> None: PaymentProcessedEventHandler: PaymentProcessedEventHandler(), OrderShippedEventHandler: OrderShippedEventHandler(), } - # Create sagas with this container and storage self._sagas = { OrderSaga: OrderSaga(), # type: ignore[arg-type] FailingOrderSaga: FailingOrderSaga(), # type: ignore[arg-type] @@ -60,15 +57,12 @@ def __init__(self, storage: SqlAlchemySagaStorage) -> None: @property def external_container(self) -> typing.Any: - """Return external container (for Container protocol compatibility).""" return self._external_container def attach_external_container(self, container: typing.Any) -> None: - """Attach external container (for Container protocol compatibility).""" self._external_container = container async def resolve(self, type_) -> typing.Any: - """Resolve type from container.""" if type_ in self._step_handlers: return self._step_handlers[type_] if type_ in self._event_handlers: @@ -82,21 +76,19 @@ async def resolve(self, type_) -> typing.Any: @pytest.fixture def storage( - saga_session_factory: async_sessionmaker[AsyncSession], + saga_session_factory_mysql: async_sessionmaker[AsyncSession], ) -> SqlAlchemySagaStorage: - """Create SqlAlchemySagaStorage instance.""" - return SqlAlchemySagaStorage(saga_session_factory) + """Create SqlAlchemySagaStorage instance (MySQL).""" + return SqlAlchemySagaStorage(saga_session_factory_mysql) @pytest.fixture def container(storage: SqlAlchemySagaStorage) -> _TestContainer: """Create test container.""" container = _TestContainer(storage) - # Clear events in step handlers before each test for step_handler in container._step_handlers.values(): if hasattr(step_handler, "_events"): step_handler._events.clear() - # Clear events in event handlers before each test for event_handler in container._event_handlers.values(): if hasattr(event_handler, "handled_events"): event_handler.handled_events.clear() @@ -108,7 +100,7 @@ def saga_mediator( container: _TestContainer, storage: SqlAlchemySagaStorage, ) -> cqrs.SagaMediator: - """Create SagaMediator with SqlAlchemySagaStorage.""" + """Create SagaMediator with SqlAlchemySagaStorage (MySQL).""" def saga_mapper(mapper: SagaMap) -> None: mapper.bind(OrderContext, OrderSaga) @@ -118,23 +110,17 @@ def events_mapper(mapper: events.EventMap) -> None: mapper.bind(PaymentProcessedEvent, PaymentProcessedEventHandler) mapper.bind(OrderShippedEvent, OrderShippedEventHandler) - # Create event emitter event_map = events.EventMap() events_mapper(event_map) - message_broker = mock.AsyncMock() message_broker.produce = mock.AsyncMock() - event_emitter = events.EventEmitter( event_map=event_map, container=container, # type: ignore message_broker=message_broker, ) - - # Create mediator directly saga_map = SagaMap() saga_mapper(saga_map) - mediator = cqrs.SagaMediator( saga_map=saga_map, container=container, # type: ignore @@ -144,48 +130,30 @@ def events_mapper(mapper: events.EventMap) -> None: concurrent_event_handle_enable=True, storage=storage, ) - return mediator -class TestSagaMediatorSqlAlchemyStorage: - """Integration tests for SagaMediator with SqlAlchemySagaStorage.""" +class TestSagaMediatorSqlAlchemyStorageMysql: + """Integration tests for SagaMediator with SqlAlchemySagaStorage (MySQL).""" async def test_saga_mediator_executes_saga_successfully( self, saga_mediator: cqrs.SagaMediator, storage: SqlAlchemySagaStorage, ) -> None: - """Test that SagaMediator executes saga successfully with SQLAlchemy storage.""" context = OrderContext(order_id="123", user_id="user1", amount=100.0) saga_id = uuid.uuid4() - step_results = [] async for result in saga_mediator.stream(context, saga_id=saga_id): step_results.append(result) - - # Verify all steps were executed assert len(step_results) == 3 - - # Verify step results assert isinstance(step_results[0].response, ReserveInventoryResponse) assert step_results[0].response.inventory_id == "inv_123" - assert step_results[0].response.reserved is True - assert isinstance(step_results[1].response, ProcessPaymentResponse) - assert step_results[1].response.payment_id == "pay_123" - assert step_results[1].response.charged is True - assert isinstance(step_results[2].response, ShipOrderResponse) - assert step_results[2].response.shipment_id == "ship_123" - assert step_results[2].response.shipped is True - - # Verify step types assert step_results[0].step_type == ReserveInventoryStep assert step_results[1].step_type == ProcessPaymentStep assert step_results[2].step_type == ShipOrderStep - - # Verify saga status in storage status, stored_context, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.COMPLETED @@ -194,24 +162,14 @@ async def test_saga_mediator_processes_events_from_steps( saga_mediator: cqrs.SagaMediator, container: _TestContainer, ) -> None: - """Test that SagaMediator processes events from steps.""" context = OrderContext(order_id="456", user_id="user2", amount=200.0) - step_results = [] async for result in saga_mediator.stream(context): step_results.append(result) - - # Verify step results were returned assert len(step_results) == 3 - assert isinstance(step_results[0].response, ReserveInventoryResponse) - assert isinstance(step_results[1].response, ProcessPaymentResponse) - assert isinstance(step_results[2].response, ShipOrderResponse) - - # Verify event handlers were called (events are processed internally) inventory_handler = await container.resolve(InventoryReservedEventHandler) payment_handler = await container.resolve(PaymentProcessedEventHandler) shipping_handler = await container.resolve(OrderShippedEventHandler) - assert len(inventory_handler.handled_events) >= 1 assert len(payment_handler.handled_events) >= 1 assert len(shipping_handler.handled_events) >= 1 @@ -221,18 +179,12 @@ async def test_saga_mediator_emits_events( saga_mediator: cqrs.SagaMediator, container: _TestContainer, ) -> None: - """Test that SagaMediator processes events via EventEmitter.""" context = OrderContext(order_id="789", user_id="user3", amount=300.0) - - step_results = [] async for result in saga_mediator.stream(context): - step_results.append(result) - - # Verify events were processed + pass inventory_handler = await container.resolve(InventoryReservedEventHandler) payment_handler = await container.resolve(PaymentProcessedEventHandler) shipping_handler = await container.resolve(OrderShippedEventHandler) - assert len(inventory_handler.handled_events) >= 1 assert len(payment_handler.handled_events) >= 1 assert len(shipping_handler.handled_events) >= 1 @@ -242,22 +194,17 @@ async def test_saga_mediator_handles_saga_failure_with_compensation( container: _TestContainer, storage: SqlAlchemySagaStorage, ) -> None: - """Test that SagaMediator handles saga failure and compensation.""" - - # Create mediator with failing saga def failing_saga_mapper(mapper: SagaMap) -> None: mapper.bind(OrderContext, FailingOrderSaga) saga_map = SagaMap() failing_saga_mapper(saga_map) - event_map = events.EventMap() event_emitter = events.EventEmitter( event_map=event_map, container=container, # type: ignore message_broker=mock.AsyncMock(), ) - failing_mediator = cqrs.SagaMediator( saga_map=saga_map, container=container, # type: ignore @@ -265,25 +212,17 @@ def failing_saga_mapper(mapper: SagaMap) -> None: event_map=event_map, storage=storage, ) - context = OrderContext(order_id="fail_123", user_id="user4", amount=400.0) saga_id = uuid.uuid4() - step_results = [] with pytest.raises(ValueError, match="Step failed for order fail_123"): async for result in failing_mediator.stream(context, saga_id=saga_id): step_results.append(result) - - # Verify that some steps were executed before failure assert len(step_results) >= 1 - - # Verify compensation was called reserve_step = await container.resolve(ReserveInventoryStep) payment_step = await container.resolve(ProcessPaymentStep) assert reserve_step.compensate_called assert payment_step.compensate_called - - # Verify saga status is FAILED status, _, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.FAILED @@ -292,37 +231,21 @@ async def test_saga_mediator_with_saga_id_recovery( saga_mediator: cqrs.SagaMediator, storage: SqlAlchemySagaStorage, ) -> None: - """Test that SagaMediator can recover saga using saga_id.""" context = OrderContext(order_id="recover_123", user_id="user5", amount=500.0) saga_id = uuid.uuid4() - - # Execute first part of saga step_results_1 = [] async for result in saga_mediator.stream(context, saga_id=saga_id): step_results_1.append(result) - # Simulate interruption after first step if len(step_results_1) == 1: break - - # Verify first step was executed assert len(step_results_1) == 1 assert isinstance(step_results_1[0].response, ReserveInventoryResponse) - assert step_results_1[0].step_type == ReserveInventoryStep - - # Verify saga is in RUNNING status status, _, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.RUNNING - - # Resume saga execution with same saga_id step_results_2 = [] async for result in saga_mediator.stream(context, saga_id=saga_id): step_results_2.append(result) - - # Verify remaining steps were executed - # Note: Saga will skip already completed steps - assert len(step_results_2) >= 2 # At least 2 more steps - - # Verify final status + assert len(step_results_2) >= 2 final_status, _, version = await storage.load_saga_state(saga_id) assert final_status == SagaStatus.COMPLETED @@ -330,9 +253,8 @@ async def test_saga_mediator_persistence_across_sessions( self, container: _TestContainer, storage: SqlAlchemySagaStorage, - saga_session_factory: async_sessionmaker[AsyncSession], + saga_session_factory_mysql: async_sessionmaker[AsyncSession], ) -> None: - """Test that saga state persists across different storage instances.""" context = OrderContext(order_id="persist_123", user_id="user6", amount=600.0) saga_id = uuid.uuid4() @@ -341,14 +263,12 @@ def saga_mapper(mapper: SagaMap) -> None: saga_map = SagaMap() saga_mapper(saga_map) - event_map = events.EventMap() event_emitter = events.EventEmitter( event_map=event_map, container=container, # type: ignore message_broker=mock.AsyncMock(), ) - mediator = cqrs.SagaMediator( saga_map=saga_map, container=container, # type: ignore @@ -356,19 +276,14 @@ def saga_mapper(mapper: SagaMap) -> None: event_map=event_map, storage=storage, ) - - # Execute first step step_results_1 = [] async for result in mediator.stream(context, saga_id=saga_id): step_results_1.append(result) if len(step_results_1) == 1: break - - # Create new storage instance and verify persistence - new_storage = SqlAlchemySagaStorage(saga_session_factory) + new_storage = SqlAlchemySagaStorage(saga_session_factory_mysql) status, stored_context, version = await new_storage.load_saga_state(saga_id) assert status == SagaStatus.RUNNING - history = await new_storage.get_step_history(saga_id) assert len(history) >= 1 assert history[0].step_name == "ReserveInventoryStep" @@ -378,33 +293,46 @@ async def test_saga_mediator_concurrent_sagas( saga_mediator: cqrs.SagaMediator, storage: SqlAlchemySagaStorage, ) -> None: - """Test that SagaMediator handles multiple concurrent sagas.""" - contexts = [ - OrderContext(order_id=f"order_{i}", user_id=f"user_{i}", amount=100.0 * i) - for i in range(3) - ] - saga_ids = [uuid.uuid4() for _ in range(3)] - - # Execute sagas concurrently import asyncio + contexts = [OrderContext(order_id=f"order_{i}", user_id=f"user_{i}", amount=100.0 * i) for i in range(3)] + saga_ids = [uuid.uuid4() for _ in range(3)] + async def execute_saga(context: OrderContext, saga_id: uuid.UUID) -> list: results = [] async for result in saga_mediator.stream(context, saga_id=saga_id): results.append(result) return results - tasks = [ - execute_saga(context, saga_id) - for context, saga_id in zip(contexts, saga_ids) - ] + tasks = [execute_saga(context, saga_id) for context, saga_id in zip(contexts, saga_ids)] all_results = await asyncio.gather(*tasks) - - # Verify all sagas completed assert len(all_results) == 3 assert all(len(results) == 3 for results in all_results) - - # Verify all sagas are in COMPLETED status for saga_id in saga_ids: status, _, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.COMPLETED + + async def test_saga_mediator_concurrent_saga_creation_no_deadlock( + self, + saga_mediator: cqrs.SagaMediator, + storage: SqlAlchemySagaStorage, + ) -> None: + import asyncio + + n = 10 + contexts = [OrderContext(order_id=f"order_{i}", user_id=f"user_{i}", amount=100.0 * (i + 1)) for i in range(n)] + saga_ids = [uuid.uuid4() for _ in range(n)] + + async def execute_saga(context: OrderContext, saga_id: uuid.UUID) -> list: + results = [] + async for result in saga_mediator.stream(context, saga_id=saga_id): + results.append(result) + return results + + tasks = [execute_saga(context, saga_id) for context, saga_id in zip(contexts, saga_ids)] + all_results = await asyncio.gather(*tasks) + assert len(all_results) == n + assert all(len(results) == 3 for results in all_results) + for saga_id in saga_ids: + status, _, _ = await storage.load_saga_state(saga_id) + assert status == SagaStatus.COMPLETED diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py new file mode 100644 index 0000000..57605c9 --- /dev/null +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -0,0 +1,338 @@ +"""Integration tests for SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" + +import typing +import uuid +from unittest import mock + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +import cqrs +from cqrs import events +from cqrs.requests.map import SagaMap +from cqrs.saga.storage.enums import SagaStatus +from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage + +from tests.integration.test_saga_mediator_memory import ( + FailingOrderSaga, + FailingStep, + InventoryReservedEvent, + InventoryReservedEventHandler, + OrderContext, + OrderShippedEvent, + OrderShippedEventHandler, + OrderSaga, + PaymentProcessedEvent, + PaymentProcessedEventHandler, + ProcessPaymentResponse, + ProcessPaymentStep, + ReserveInventoryResponse, + ReserveInventoryStep, + ShipOrderResponse, + ShipOrderStep, +) + + +class _TestContainer: + """Test container that resolves step handlers, sagas, and event handlers.""" + + def __init__(self, storage: SqlAlchemySagaStorage) -> None: + self._storage = storage + self._external_container = None + self._step_handlers = { + ReserveInventoryStep: ReserveInventoryStep(), + ProcessPaymentStep: ProcessPaymentStep(), + ShipOrderStep: ShipOrderStep(), + FailingStep: FailingStep(), + } + self._event_handlers = { + InventoryReservedEventHandler: InventoryReservedEventHandler(), + PaymentProcessedEventHandler: PaymentProcessedEventHandler(), + OrderShippedEventHandler: OrderShippedEventHandler(), + } + self._sagas = { + OrderSaga: OrderSaga(), # type: ignore[arg-type] + FailingOrderSaga: FailingOrderSaga(), # type: ignore[arg-type] + } + + @property + def external_container(self) -> typing.Any: + return self._external_container + + def attach_external_container(self, container: typing.Any) -> None: + self._external_container = container + + async def resolve(self, type_) -> typing.Any: + if type_ in self._step_handlers: + return self._step_handlers[type_] + if type_ in self._event_handlers: + return self._event_handlers[type_] + if type_ in self._sagas: + return self._sagas[type_] + if type_ == SqlAlchemySagaStorage: + return self._storage + raise ValueError(f"Unknown type: {type_}") + + +@pytest.fixture +def storage( + saga_session_factory_postgres: async_sessionmaker[AsyncSession], +) -> SqlAlchemySagaStorage: + """Create SqlAlchemySagaStorage instance (PostgreSQL).""" + return SqlAlchemySagaStorage(saga_session_factory_postgres) + + +@pytest.fixture +def container(storage: SqlAlchemySagaStorage) -> _TestContainer: + """Create test container.""" + container = _TestContainer(storage) + for step_handler in container._step_handlers.values(): + if hasattr(step_handler, "_events"): + step_handler._events.clear() + for event_handler in container._event_handlers.values(): + if hasattr(event_handler, "handled_events"): + event_handler.handled_events.clear() + return container + + +@pytest.fixture +def saga_mediator( + container: _TestContainer, + storage: SqlAlchemySagaStorage, +) -> cqrs.SagaMediator: + """Create SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" + + def saga_mapper(mapper: SagaMap) -> None: + mapper.bind(OrderContext, OrderSaga) + + def events_mapper(mapper: events.EventMap) -> None: + mapper.bind(InventoryReservedEvent, InventoryReservedEventHandler) + mapper.bind(PaymentProcessedEvent, PaymentProcessedEventHandler) + mapper.bind(OrderShippedEvent, OrderShippedEventHandler) + + event_map = events.EventMap() + events_mapper(event_map) + message_broker = mock.AsyncMock() + message_broker.produce = mock.AsyncMock() + event_emitter = events.EventEmitter( + event_map=event_map, + container=container, # type: ignore + message_broker=message_broker, + ) + saga_map = SagaMap() + saga_mapper(saga_map) + mediator = cqrs.SagaMediator( + saga_map=saga_map, + container=container, # type: ignore + event_emitter=event_emitter, + event_map=event_map, + max_concurrent_event_handlers=2, + concurrent_event_handle_enable=True, + storage=storage, + ) + return mediator + + +class TestSagaMediatorSqlAlchemyStoragePostgres: + """Integration tests for SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" + + async def test_saga_mediator_executes_saga_successfully( + self, + saga_mediator: cqrs.SagaMediator, + storage: SqlAlchemySagaStorage, + ) -> None: + context = OrderContext(order_id="123", user_id="user1", amount=100.0) + saga_id = uuid.uuid4() + step_results = [] + async for result in saga_mediator.stream(context, saga_id=saga_id): + step_results.append(result) + assert len(step_results) == 3 + assert isinstance(step_results[0].response, ReserveInventoryResponse) + assert step_results[0].response.inventory_id == "inv_123" + assert isinstance(step_results[1].response, ProcessPaymentResponse) + assert isinstance(step_results[2].response, ShipOrderResponse) + assert step_results[0].step_type == ReserveInventoryStep + assert step_results[1].step_type == ProcessPaymentStep + assert step_results[2].step_type == ShipOrderStep + status, stored_context, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.COMPLETED + + async def test_saga_mediator_processes_events_from_steps( + self, + saga_mediator: cqrs.SagaMediator, + container: _TestContainer, + ) -> None: + context = OrderContext(order_id="456", user_id="user2", amount=200.0) + step_results = [] + async for result in saga_mediator.stream(context): + step_results.append(result) + assert len(step_results) == 3 + inventory_handler = await container.resolve(InventoryReservedEventHandler) + payment_handler = await container.resolve(PaymentProcessedEventHandler) + shipping_handler = await container.resolve(OrderShippedEventHandler) + assert len(inventory_handler.handled_events) >= 1 + assert len(payment_handler.handled_events) >= 1 + assert len(shipping_handler.handled_events) >= 1 + + async def test_saga_mediator_emits_events( + self, + saga_mediator: cqrs.SagaMediator, + container: _TestContainer, + ) -> None: + context = OrderContext(order_id="789", user_id="user3", amount=300.0) + async for result in saga_mediator.stream(context): + pass + inventory_handler = await container.resolve(InventoryReservedEventHandler) + payment_handler = await container.resolve(PaymentProcessedEventHandler) + shipping_handler = await container.resolve(OrderShippedEventHandler) + assert len(inventory_handler.handled_events) >= 1 + assert len(payment_handler.handled_events) >= 1 + assert len(shipping_handler.handled_events) >= 1 + + async def test_saga_mediator_handles_saga_failure_with_compensation( + self, + container: _TestContainer, + storage: SqlAlchemySagaStorage, + ) -> None: + def failing_saga_mapper(mapper: SagaMap) -> None: + mapper.bind(OrderContext, FailingOrderSaga) + + saga_map = SagaMap() + failing_saga_mapper(saga_map) + event_map = events.EventMap() + event_emitter = events.EventEmitter( + event_map=event_map, + container=container, # type: ignore + message_broker=mock.AsyncMock(), + ) + failing_mediator = cqrs.SagaMediator( + saga_map=saga_map, + container=container, # type: ignore + event_emitter=event_emitter, + event_map=event_map, + storage=storage, + ) + context = OrderContext(order_id="fail_123", user_id="user4", amount=400.0) + saga_id = uuid.uuid4() + step_results = [] + with pytest.raises(ValueError, match="Step failed for order fail_123"): + async for result in failing_mediator.stream(context, saga_id=saga_id): + step_results.append(result) + assert len(step_results) >= 1 + reserve_step = await container.resolve(ReserveInventoryStep) + payment_step = await container.resolve(ProcessPaymentStep) + assert reserve_step.compensate_called + assert payment_step.compensate_called + status, _, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.FAILED + + async def test_saga_mediator_with_saga_id_recovery( + self, + saga_mediator: cqrs.SagaMediator, + storage: SqlAlchemySagaStorage, + ) -> None: + context = OrderContext(order_id="recover_123", user_id="user5", amount=500.0) + saga_id = uuid.uuid4() + step_results_1 = [] + async for result in saga_mediator.stream(context, saga_id=saga_id): + step_results_1.append(result) + if len(step_results_1) == 1: + break + assert len(step_results_1) == 1 + assert isinstance(step_results_1[0].response, ReserveInventoryResponse) + status, _, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.RUNNING + step_results_2 = [] + async for result in saga_mediator.stream(context, saga_id=saga_id): + step_results_2.append(result) + assert len(step_results_2) >= 2 + final_status, _, version = await storage.load_saga_state(saga_id) + assert final_status == SagaStatus.COMPLETED + + async def test_saga_mediator_persistence_across_sessions( + self, + container: _TestContainer, + storage: SqlAlchemySagaStorage, + saga_session_factory_postgres: async_sessionmaker[AsyncSession], + ) -> None: + context = OrderContext(order_id="persist_123", user_id="user6", amount=600.0) + saga_id = uuid.uuid4() + + def saga_mapper(mapper: SagaMap) -> None: + mapper.bind(OrderContext, OrderSaga) + + saga_map = SagaMap() + saga_mapper(saga_map) + event_map = events.EventMap() + event_emitter = events.EventEmitter( + event_map=event_map, + container=container, # type: ignore + message_broker=mock.AsyncMock(), + ) + mediator = cqrs.SagaMediator( + saga_map=saga_map, + container=container, # type: ignore + event_emitter=event_emitter, + event_map=event_map, + storage=storage, + ) + step_results_1 = [] + async for result in mediator.stream(context, saga_id=saga_id): + step_results_1.append(result) + if len(step_results_1) == 1: + break + new_storage = SqlAlchemySagaStorage(saga_session_factory_postgres) + status, stored_context, version = await new_storage.load_saga_state(saga_id) + assert status == SagaStatus.RUNNING + history = await new_storage.get_step_history(saga_id) + assert len(history) >= 1 + assert history[0].step_name == "ReserveInventoryStep" + + async def test_saga_mediator_concurrent_sagas( + self, + saga_mediator: cqrs.SagaMediator, + storage: SqlAlchemySagaStorage, + ) -> None: + import asyncio + + contexts = [OrderContext(order_id=f"order_{i}", user_id=f"user_{i}", amount=100.0 * i) for i in range(3)] + saga_ids = [uuid.uuid4() for _ in range(3)] + + async def execute_saga(context: OrderContext, saga_id: uuid.UUID) -> list: + results = [] + async for result in saga_mediator.stream(context, saga_id=saga_id): + results.append(result) + return results + + tasks = [execute_saga(context, saga_id) for context, saga_id in zip(contexts, saga_ids)] + all_results = await asyncio.gather(*tasks) + assert len(all_results) == 3 + assert all(len(results) == 3 for results in all_results) + for saga_id in saga_ids: + status, _, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.COMPLETED + + async def test_saga_mediator_concurrent_saga_creation_no_deadlock( + self, + saga_mediator: cqrs.SagaMediator, + storage: SqlAlchemySagaStorage, + ) -> None: + import asyncio + + n = 10 + contexts = [OrderContext(order_id=f"order_{i}", user_id=f"user_{i}", amount=100.0 * (i + 1)) for i in range(n)] + saga_ids = [uuid.uuid4() for _ in range(n)] + + async def execute_saga(context: OrderContext, saga_id: uuid.UUID) -> list: + results = [] + async for result in saga_mediator.stream(context, saga_id=saga_id): + results.append(result) + return results + + tasks = [execute_saga(context, saga_id) for context, saga_id in zip(contexts, saga_ids)] + all_results = await asyncio.gather(*tasks) + assert len(all_results) == n + assert all(len(results) == 3 for results in all_results) + for saga_id in saga_ids: + status, _, _ = await storage.load_saga_state(saga_id) + assert status == SagaStatus.COMPLETED diff --git a/tests/integration/test_saga_storage_sqlalchemy.py b/tests/integration/test_saga_storage_sqlalchemy_mysql.py similarity index 63% rename from tests/integration/test_saga_storage_sqlalchemy.py rename to tests/integration/test_saga_storage_sqlalchemy_mysql.py index 3d7e5fd..7f691e7 100644 --- a/tests/integration/test_saga_storage_sqlalchemy.py +++ b/tests/integration/test_saga_storage_sqlalchemy_mysql.py @@ -1,4 +1,6 @@ -"""Integration tests for SqlAlchemySagaStorage.""" +"""Integration tests for SqlAlchemySagaStorage (MySQL). +Uses DATABASE_DSN_MYSQL from fixtures (pytest-config.ini / env). +""" import asyncio import uuid @@ -16,31 +18,27 @@ SqlAlchemySagaStorage, ) -# Fixtures init_saga_orm and saga_session_factory are imported from tests/integration/fixtures.py - @pytest.fixture def storage( - saga_session_factory: async_sessionmaker[AsyncSession], + saga_session_factory_mysql: async_sessionmaker[AsyncSession], ) -> SqlAlchemySagaStorage: - """Create a SqlAlchemySagaStorage instance for each test.""" - return SqlAlchemySagaStorage(saga_session_factory) + """SqlAlchemySagaStorage для MySQL (фикстура init_saga_orm_mysql поднимает схему).""" + return SqlAlchemySagaStorage(saga_session_factory_mysql) @pytest.fixture def saga_id() -> uuid.UUID: - """Generate a test saga ID.""" return uuid.uuid4() @pytest.fixture def test_context() -> dict[str, str]: - """Test context data.""" return {"order_id": "123", "user_id": "user1", "amount": "100.0"} -class TestIntegration: - """Integration tests for multiple operations.""" +class TestIntegrationMysql: + """Integration tests for multiple operations (MySQL).""" async def test_full_saga_lifecycle( self, @@ -48,18 +46,8 @@ async def test_full_saga_lifecycle( saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Test complete saga lifecycle with all operations.""" - # Create saga (storage handles transaction commit internally) - await storage.create_saga( - saga_id=saga_id, - name="order_saga", - context=test_context, - ) - - # Update status to running + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) await storage.update_status(saga_id=saga_id, status=SagaStatus.RUNNING) - - # Log step executions await storage.log_step( saga_id=saga_id, step_name="reserve_inventory", @@ -84,32 +72,17 @@ async def test_full_saga_lifecycle( action="act", status=SagaStepStatus.COMPLETED, ) - - # Update context updated_context = {**test_context, "payment_id": "pay_123"} await storage.update_context(saga_id=saga_id, context=updated_context) - - # Update status to completed await storage.update_status(saga_id=saga_id, status=SagaStatus.COMPLETED) - - # Verify final state status, context, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.COMPLETED assert context == updated_context - # Initial create(1) + update_status(RUNNING)(2) + update_context(3) + update_status(COMPLETED)(4) = 4 assert version == 4 - - # Verify history history = await storage.get_step_history(saga_id) assert len(history) == 4 assert history[0].step_name == "reserve_inventory" - assert history[0].status == SagaStepStatus.STARTED - assert history[1].step_name == "reserve_inventory" - assert history[1].status == SagaStepStatus.COMPLETED assert history[2].step_name == "process_payment" - assert history[2].status == SagaStepStatus.STARTED - assert history[3].step_name == "process_payment" - assert history[3].status == SagaStepStatus.COMPLETED async def test_compensation_scenario( self, @@ -117,14 +90,7 @@ async def test_compensation_scenario( saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Test saga compensation scenario.""" - await storage.create_saga( - saga_id=saga_id, - name="order_saga", - context=test_context, - ) - - # Log successful steps + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) await storage.log_step( saga_id=saga_id, step_name="reserve_inventory", @@ -137,11 +103,7 @@ async def test_compensation_scenario( action="act", status=SagaStepStatus.COMPLETED, ) - - # Update status to compensating await storage.update_status(saga_id=saga_id, status=SagaStatus.COMPENSATING) - - # Log compensation steps await storage.log_step( saga_id=saga_id, step_name="process_payment", @@ -156,60 +118,33 @@ async def test_compensation_scenario( status=SagaStepStatus.COMPENSATED, details="Inventory released", ) - - # Update status to failed await storage.update_status(saga_id=saga_id, status=SagaStatus.FAILED) - - # Verify state status, context, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.FAILED - # Initial create(1) + update_status(COMPENSATING)(2) + update_status(FAILED)(3) = 3 assert version == 3 - - # Verify history history = await storage.get_step_history(saga_id) assert len(history) == 4 - assert history[0].action == "act" - assert history[1].action == "act" assert history[2].action == "compensate" assert history[3].action == "compensate" - assert history[2].details == "Payment refunded" - assert history[3].details == "Inventory released" async def test_persistence_across_sessions( self, - saga_session_factory: async_sessionmaker[AsyncSession], + saga_session_factory_mysql: async_sessionmaker[AsyncSession], saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Test that saga state persists across different storage instances.""" - # Create saga with first storage instance - storage1 = SqlAlchemySagaStorage(saga_session_factory) - await storage1.create_saga( - saga_id=saga_id, - name="order_saga", - context=test_context, - ) + storage1 = SqlAlchemySagaStorage(saga_session_factory_mysql) + await storage1.create_saga(saga_id=saga_id, name="order_saga", context=test_context) await storage1.update_status(saga_id=saga_id, status=SagaStatus.RUNNING) - await storage1.log_step( - saga_id=saga_id, - step_name="step1", - action="act", - status=SagaStepStatus.COMPLETED, - ) - - # Create new storage instance and verify persistence - # Note: Since storage now commits internally, data is already persisted - storage2 = SqlAlchemySagaStorage(saga_session_factory) + await storage1.log_step(saga_id=saga_id, step_name="step1", action="act", status=SagaStepStatus.COMPLETED) + storage2 = SqlAlchemySagaStorage(saga_session_factory_mysql) status, context, version = await storage2.load_saga_state(saga_id) assert status == SagaStatus.RUNNING assert context == test_context - assert version == 2 # create + update_status - + assert version == 2 history = await storage2.get_step_history(saga_id) assert len(history) == 1 assert history[0].step_name == "step1" - assert history[0].status == SagaStepStatus.COMPLETED async def test_concurrent_updates( self, @@ -217,20 +152,11 @@ async def test_concurrent_updates( saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Test handling of multiple sequential updates.""" - await storage.create_saga( - saga_id=saga_id, - name="order_saga", - context=test_context, - ) - - # Perform multiple updates + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) await storage.update_status(saga_id=saga_id, status=SagaStatus.RUNNING) await storage.update_context(saga_id=saga_id, context={"updated": "context1"}) await storage.update_status(saga_id=saga_id, status=SagaStatus.COMPENSATING) await storage.update_context(saga_id=saga_id, context={"updated": "context2"}) - - # Verify final state status, context, version = await storage.load_saga_state(saga_id) assert status == SagaStatus.COMPENSATING assert context == {"updated": "context2"} @@ -242,85 +168,58 @@ async def test_optimistic_locking( saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Test that optimistic locking prevents concurrent modifications.""" - await storage.create_saga( - saga_id=saga_id, - name="order_saga", - context=test_context, - ) - - # Get initial state + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) _, _, version = await storage.load_saga_state(saga_id) assert version == 1 - - # Successful update with correct version new_context = {**test_context, "updated": True} await storage.update_context(saga_id, new_context, current_version=version) - - # Verify version incremented _, _, new_version = await storage.load_saga_state(saga_id) assert new_version == 2 - - # Failed update with old version with pytest.raises(SagaConcurrencyError): - await storage.update_context( - saga_id, - {"stale": True}, - current_version=version, # Using old version 1 - ) - - # State should not have changed + await storage.update_context(saga_id, {"stale": True}, current_version=version) _, final_context, final_version = await storage.load_saga_state(saga_id) assert final_context == new_context assert final_version == 2 -class TestRecoverySqlAlchemy: - """Integration tests for get_sagas_for_recovery and increment_recovery_attempts (SqlAlchemy).""" +class TestRecoverySqlAlchemyMysql: + """Integration tests for get_sagas_for_recovery and increment_recovery_attempts (MySQL).""" @pytest.fixture(autouse=True) async def _clean_saga_tables( self, - saga_session_factory: async_sessionmaker[AsyncSession], + saga_session_factory_mysql: async_sessionmaker[AsyncSession], ) -> AsyncGenerator[None, None]: - """Clear saga tables before each test so get_sagas_for_recovery sees only this test's data.""" - async with saga_session_factory() as session: + async with saga_session_factory_mysql() as session: await session.execute(delete(SagaLogModel)) await session.execute(delete(SagaExecutionModel)) await session.commit() yield - # --- get_sagas_for_recovery: positive --- - async def test_get_sagas_for_recovery_returns_recoverable_sagas( self, storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Positive: returns RUNNING and COMPENSATING sagas only; FAILED excluded.""" id1, id2, id3 = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() for sid in (id1, id2, id3): await storage.create_saga(saga_id=sid, name="saga", context=test_context) await storage.update_status(id1, SagaStatus.RUNNING) await storage.update_status(id2, SagaStatus.COMPENSATING) await storage.update_status(id3, SagaStatus.FAILED) - ids = await storage.get_sagas_for_recovery(limit=10) assert set(ids) == {id1, id2} assert id3 not in ids - assert len(ids) == 2 async def test_get_sagas_for_recovery_respects_limit( self, storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Positive: returns at most `limit` saga IDs.""" for _ in range(5): sid = uuid.uuid4() await storage.create_saga(saga_id=sid, name="saga", context=test_context) await storage.update_status(sid, SagaStatus.RUNNING) - ids = await storage.get_sagas_for_recovery(limit=2) assert len(ids) == 2 @@ -329,16 +228,13 @@ async def test_get_sagas_for_recovery_respects_max_recovery_attempts( storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Positive: only returns sagas with recovery_attempts < max_recovery_attempts.""" - id_low = uuid.uuid4() - id_high = uuid.uuid4() + id_low, id_high = uuid.uuid4(), uuid.uuid4() await storage.create_saga(saga_id=id_low, name="saga", context=test_context) await storage.create_saga(saga_id=id_high, name="saga", context=test_context) await storage.update_status(id_low, SagaStatus.RUNNING) await storage.update_status(id_high, SagaStatus.RUNNING) for _ in range(5): await storage.increment_recovery_attempts(id_high) - ids = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) assert id_low in ids assert id_high not in ids @@ -348,15 +244,12 @@ async def test_get_sagas_for_recovery_ordered_by_updated_at( storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Positive: result ordered by updated_at ascending (oldest first).""" id1, id2, id3 = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() for sid in (id1, id2, id3): await storage.create_saga(saga_id=sid, name="saga", context=test_context) await storage.update_status(sid, SagaStatus.RUNNING) - # Ensure id2 has a strictly later updated_at (DB may use second precision). await asyncio.sleep(1.0) await storage.update_context(id2, {**test_context, "touched": True}) - ids = await storage.get_sagas_for_recovery(limit=10) assert len(ids) == 3 assert ids[-1] == id2 @@ -366,14 +259,10 @@ async def test_get_sagas_for_recovery_stale_after_excludes_recently_updated( storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Positive: with stale_after_seconds, recently updated sagas are excluded.""" id_recent = uuid.uuid4() await storage.create_saga(saga_id=id_recent, name="saga", context=test_context) await storage.update_status(id_recent, SagaStatus.RUNNING) - ids = await storage.get_sagas_for_recovery( - limit=10, - stale_after_seconds=999999, - ) + ids = await storage.get_sagas_for_recovery(limit=10, stale_after_seconds=999999) assert id_recent not in ids async def test_get_sagas_for_recovery_without_stale_after_unchanged_behavior( @@ -381,7 +270,6 @@ async def test_get_sagas_for_recovery_without_stale_after_unchanged_behavior( storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Backward compat: without stale_after_seconds, recently updated sagas are included.""" sid = uuid.uuid4() await storage.create_saga(saga_id=sid, name="saga", context=test_context) await storage.update_status(sid, SagaStatus.RUNNING) @@ -393,55 +281,26 @@ async def test_get_sagas_for_recovery_filters_by_saga_name_when_provided( storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Positive: when saga_name is set, only sagas with that name are returned.""" - id_foo1 = uuid.uuid4() - id_foo2 = uuid.uuid4() - id_bar = uuid.uuid4() - await storage.create_saga( - saga_id=id_foo1, - name="OrderSaga", - context=test_context, - ) - await storage.create_saga( - saga_id=id_foo2, - name="OrderSaga", - context=test_context, - ) - await storage.create_saga( - saga_id=id_bar, - name="PaymentSaga", - context=test_context, - ) + id_foo1, id_foo2, id_bar = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + await storage.create_saga(saga_id=id_foo1, name="OrderSaga", context=test_context) + await storage.create_saga(saga_id=id_foo2, name="OrderSaga", context=test_context) + await storage.create_saga(saga_id=id_bar, name="PaymentSaga", context=test_context) await storage.update_status(id_foo1, SagaStatus.RUNNING) await storage.update_status(id_foo2, SagaStatus.RUNNING) await storage.update_status(id_bar, SagaStatus.RUNNING) - - ids_all = await storage.get_sagas_for_recovery(limit=10) - assert len(ids_all) == 3 - ids_order = await storage.get_sagas_for_recovery( - limit=10, - saga_name="OrderSaga", - ) + ids_order = await storage.get_sagas_for_recovery(limit=10, saga_name="OrderSaga") assert set(ids_order) == {id_foo1, id_foo2} - ids_payment = await storage.get_sagas_for_recovery( - limit=10, - saga_name="PaymentSaga", - ) + ids_payment = await storage.get_sagas_for_recovery(limit=10, saga_name="PaymentSaga") assert ids_payment == [id_bar] - ids_nonexistent = await storage.get_sagas_for_recovery( - limit=10, - saga_name="NonExistentSaga", - ) - assert ids_nonexistent == [] + ids_none = await storage.get_sagas_for_recovery(limit=10, saga_name="NonExistentSaga") + assert ids_none == [] async def test_get_sagas_for_recovery_saga_name_none_returns_all_types( self, storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Backward compat: when saga_name is None, all saga types are returned.""" - id1 = uuid.uuid4() - id2 = uuid.uuid4() + id1, id2 = uuid.uuid4(), uuid.uuid4() await storage.create_saga(saga_id=id1, name="SagaA", context=test_context) await storage.create_saga(saga_id=id2, name="SagaB", context=test_context) await storage.update_status(id1, SagaStatus.RUNNING) @@ -449,18 +308,14 @@ async def test_get_sagas_for_recovery_saga_name_none_returns_all_types( ids = await storage.get_sagas_for_recovery(limit=10, saga_name=None) assert set(ids) == {id1, id2} - # --- get_sagas_for_recovery: negative --- - async def test_get_sagas_for_recovery_empty_when_none_recoverable( self, storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Negative: returns empty list when no recoverable sagas.""" sid = uuid.uuid4() await storage.create_saga(saga_id=sid, name="saga", context=test_context) await storage.update_status(sid, SagaStatus.COMPLETED) - ids = await storage.get_sagas_for_recovery(limit=10) assert ids == [] @@ -469,46 +324,27 @@ async def test_get_sagas_for_recovery_excludes_pending_and_completed( storage: SqlAlchemySagaStorage, test_context: dict[str, str], ) -> None: - """Negative: PENDING and COMPLETED sagas are not returned.""" - id_pending = uuid.uuid4() - id_completed = uuid.uuid4() + id_pending, id_completed = uuid.uuid4(), uuid.uuid4() await storage.create_saga(saga_id=id_pending, name="saga", context=test_context) - await storage.create_saga( - saga_id=id_completed, - name="saga", - context=test_context, - ) + await storage.create_saga(saga_id=id_completed, name="saga", context=test_context) await storage.update_status(id_completed, SagaStatus.COMPLETED) - ids = await storage.get_sagas_for_recovery(limit=10) assert id_pending not in ids assert id_completed not in ids - # --- increment_recovery_attempts: positive --- - async def test_increment_recovery_attempts_increments_counter( self, storage: SqlAlchemySagaStorage, saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Positive: recovery_attempts increases; saga drops out after max_recovery_attempts.""" await storage.create_saga(saga_id=saga_id, name="saga", context=test_context) await storage.update_status(saga_id, SagaStatus.RUNNING) - - ids_before = await storage.get_sagas_for_recovery( - limit=10, - max_recovery_attempts=5, - ) + ids_before = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) assert saga_id in ids_before - for _ in range(5): await storage.increment_recovery_attempts(saga_id) - - ids_after = await storage.get_sagas_for_recovery( - limit=10, - max_recovery_attempts=5, - ) + ids_after = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) assert saga_id not in ids_after async def test_increment_recovery_attempts_with_new_status( @@ -517,58 +353,41 @@ async def test_increment_recovery_attempts_with_new_status( saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Positive: optional new_status updates saga status.""" await storage.create_saga(saga_id=saga_id, name="saga", context=test_context) await storage.update_status(saga_id, SagaStatus.RUNNING) - await storage.increment_recovery_attempts(saga_id, new_status=SagaStatus.FAILED) status, _, _ = await storage.load_saga_state(saga_id) assert status == SagaStatus.FAILED - # --- increment_recovery_attempts: negative --- - async def test_increment_recovery_attempts_raises_when_saga_not_found( self, storage: SqlAlchemySagaStorage, ) -> None: - """Negative: raises ValueError when saga_id does not exist.""" unknown_id = uuid.uuid4() with pytest.raises(ValueError, match="not found"): await storage.increment_recovery_attempts(unknown_id) - # --- set_recovery_attempts: positive --- - async def test_set_recovery_attempts_sets_value( self, storage: SqlAlchemySagaStorage, saga_id: uuid.UUID, test_context: dict[str, str], ) -> None: - """Positive: recovery_attempts is set to the given value.""" await storage.create_saga(saga_id=saga_id, name="saga", context=test_context) await storage.update_status(saga_id, SagaStatus.RUNNING) await storage.increment_recovery_attempts(saga_id) await storage.increment_recovery_attempts(saga_id) - await storage.set_recovery_attempts(saga_id, 0) - ids_after_reset = await storage.get_sagas_for_recovery( - limit=10, - max_recovery_attempts=5, - ) - assert saga_id in ids_after_reset - + ids_reset = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert saga_id in ids_reset await storage.set_recovery_attempts(saga_id, 5) - ids_after_max = await storage.get_sagas_for_recovery( - limit=10, - max_recovery_attempts=5, - ) - assert saga_id not in ids_after_max + ids_max = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert saga_id not in ids_max async def test_set_recovery_attempts_raises_when_saga_not_found( self, storage: SqlAlchemySagaStorage, ) -> None: - """Negative: raises ValueError when saga_id does not exist.""" unknown_id = uuid.uuid4() with pytest.raises(ValueError, match="not found"): await storage.set_recovery_attempts(unknown_id, 0) diff --git a/tests/integration/test_saga_storage_sqlalchemy_postgres.py b/tests/integration/test_saga_storage_sqlalchemy_postgres.py new file mode 100644 index 0000000..9ab308c --- /dev/null +++ b/tests/integration/test_saga_storage_sqlalchemy_postgres.py @@ -0,0 +1,393 @@ +"""Integration tests for SqlAlchemySagaStorage (PostgreSQL). +Uses DATABASE_DSN_POSTGRESQL from fixtures (pytest-config.ini / env). +""" + +import asyncio +import uuid +from collections.abc import AsyncGenerator + +import pytest +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from cqrs.dispatcher.exceptions import SagaConcurrencyError +from cqrs.saga.storage.enums import SagaStatus, SagaStepStatus +from cqrs.saga.storage.sqlalchemy import ( + SagaExecutionModel, + SagaLogModel, + SqlAlchemySagaStorage, +) + + +@pytest.fixture +def storage( + saga_session_factory_postgres: async_sessionmaker[AsyncSession], +) -> SqlAlchemySagaStorage: + """SqlAlchemySagaStorage для PostgreSQL (фикстура init_saga_orm_postgres поднимает схему).""" + return SqlAlchemySagaStorage(saga_session_factory_postgres) + + +@pytest.fixture +def saga_id() -> uuid.UUID: + return uuid.uuid4() + + +@pytest.fixture +def test_context() -> dict[str, str]: + return {"order_id": "123", "user_id": "user1", "amount": "100.0"} + + +class TestIntegrationPostgres: + """Integration tests for multiple operations (PostgreSQL).""" + + async def test_full_saga_lifecycle( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) + await storage.update_status(saga_id=saga_id, status=SagaStatus.RUNNING) + await storage.log_step( + saga_id=saga_id, + step_name="reserve_inventory", + action="act", + status=SagaStepStatus.STARTED, + ) + await storage.log_step( + saga_id=saga_id, + step_name="reserve_inventory", + action="act", + status=SagaStepStatus.COMPLETED, + ) + await storage.log_step( + saga_id=saga_id, + step_name="process_payment", + action="act", + status=SagaStepStatus.STARTED, + ) + await storage.log_step( + saga_id=saga_id, + step_name="process_payment", + action="act", + status=SagaStepStatus.COMPLETED, + ) + updated_context = {**test_context, "payment_id": "pay_123"} + await storage.update_context(saga_id=saga_id, context=updated_context) + await storage.update_status(saga_id=saga_id, status=SagaStatus.COMPLETED) + status, context, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.COMPLETED + assert context == updated_context + assert version == 4 + history = await storage.get_step_history(saga_id) + assert len(history) == 4 + assert history[0].step_name == "reserve_inventory" + assert history[2].step_name == "process_payment" + + async def test_compensation_scenario( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) + await storage.log_step( + saga_id=saga_id, + step_name="reserve_inventory", + action="act", + status=SagaStepStatus.COMPLETED, + ) + await storage.log_step( + saga_id=saga_id, + step_name="process_payment", + action="act", + status=SagaStepStatus.COMPLETED, + ) + await storage.update_status(saga_id=saga_id, status=SagaStatus.COMPENSATING) + await storage.log_step( + saga_id=saga_id, + step_name="process_payment", + action="compensate", + status=SagaStepStatus.COMPENSATED, + details="Payment refunded", + ) + await storage.log_step( + saga_id=saga_id, + step_name="reserve_inventory", + action="compensate", + status=SagaStepStatus.COMPENSATED, + details="Inventory released", + ) + await storage.update_status(saga_id=saga_id, status=SagaStatus.FAILED) + status, context, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.FAILED + assert version == 3 + history = await storage.get_step_history(saga_id) + assert len(history) == 4 + assert history[2].action == "compensate" + assert history[3].action == "compensate" + + async def test_persistence_across_sessions( + self, + saga_session_factory_postgres: async_sessionmaker[AsyncSession], + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + storage1 = SqlAlchemySagaStorage(saga_session_factory_postgres) + await storage1.create_saga(saga_id=saga_id, name="order_saga", context=test_context) + await storage1.update_status(saga_id=saga_id, status=SagaStatus.RUNNING) + await storage1.log_step(saga_id=saga_id, step_name="step1", action="act", status=SagaStepStatus.COMPLETED) + storage2 = SqlAlchemySagaStorage(saga_session_factory_postgres) + status, context, version = await storage2.load_saga_state(saga_id) + assert status == SagaStatus.RUNNING + assert context == test_context + assert version == 2 + history = await storage2.get_step_history(saga_id) + assert len(history) == 1 + assert history[0].step_name == "step1" + + async def test_concurrent_updates( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) + await storage.update_status(saga_id=saga_id, status=SagaStatus.RUNNING) + await storage.update_context(saga_id=saga_id, context={"updated": "context1"}) + await storage.update_status(saga_id=saga_id, status=SagaStatus.COMPENSATING) + await storage.update_context(saga_id=saga_id, context={"updated": "context2"}) + status, context, version = await storage.load_saga_state(saga_id) + assert status == SagaStatus.COMPENSATING + assert context == {"updated": "context2"} + assert version == 5 + + async def test_optimistic_locking( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="order_saga", context=test_context) + _, _, version = await storage.load_saga_state(saga_id) + assert version == 1 + new_context = {**test_context, "updated": True} + await storage.update_context(saga_id, new_context, current_version=version) + _, _, new_version = await storage.load_saga_state(saga_id) + assert new_version == 2 + with pytest.raises(SagaConcurrencyError): + await storage.update_context(saga_id, {"stale": True}, current_version=version) + _, final_context, final_version = await storage.load_saga_state(saga_id) + assert final_context == new_context + assert final_version == 2 + + +class TestRecoverySqlAlchemyPostgres: + """Integration tests for get_sagas_for_recovery and increment_recovery_attempts (PostgreSQL).""" + + @pytest.fixture(autouse=True) + async def _clean_saga_tables( + self, + saga_session_factory_postgres: async_sessionmaker[AsyncSession], + ) -> AsyncGenerator[None, None]: + async with saga_session_factory_postgres() as session: + await session.execute(delete(SagaLogModel)) + await session.execute(delete(SagaExecutionModel)) + await session.commit() + yield + + async def test_get_sagas_for_recovery_returns_recoverable_sagas( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id1, id2, id3 = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + for sid in (id1, id2, id3): + await storage.create_saga(saga_id=sid, name="saga", context=test_context) + await storage.update_status(id1, SagaStatus.RUNNING) + await storage.update_status(id2, SagaStatus.COMPENSATING) + await storage.update_status(id3, SagaStatus.FAILED) + ids = await storage.get_sagas_for_recovery(limit=10) + assert set(ids) == {id1, id2} + assert id3 not in ids + + async def test_get_sagas_for_recovery_respects_limit( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + for _ in range(5): + sid = uuid.uuid4() + await storage.create_saga(saga_id=sid, name="saga", context=test_context) + await storage.update_status(sid, SagaStatus.RUNNING) + ids = await storage.get_sagas_for_recovery(limit=2) + assert len(ids) == 2 + + async def test_get_sagas_for_recovery_respects_max_recovery_attempts( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id_low, id_high = uuid.uuid4(), uuid.uuid4() + await storage.create_saga(saga_id=id_low, name="saga", context=test_context) + await storage.create_saga(saga_id=id_high, name="saga", context=test_context) + await storage.update_status(id_low, SagaStatus.RUNNING) + await storage.update_status(id_high, SagaStatus.RUNNING) + for _ in range(5): + await storage.increment_recovery_attempts(id_high) + ids = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert id_low in ids + assert id_high not in ids + + async def test_get_sagas_for_recovery_ordered_by_updated_at( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id1, id2, id3 = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + for sid in (id1, id2, id3): + await storage.create_saga(saga_id=sid, name="saga", context=test_context) + await storage.update_status(sid, SagaStatus.RUNNING) + await asyncio.sleep(1.0) + await storage.update_context(id2, {**test_context, "touched": True}) + ids = await storage.get_sagas_for_recovery(limit=10) + assert len(ids) == 3 + assert ids[-1] == id2 + + async def test_get_sagas_for_recovery_stale_after_excludes_recently_updated( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id_recent = uuid.uuid4() + await storage.create_saga(saga_id=id_recent, name="saga", context=test_context) + await storage.update_status(id_recent, SagaStatus.RUNNING) + ids = await storage.get_sagas_for_recovery(limit=10, stale_after_seconds=999999) + assert id_recent not in ids + + async def test_get_sagas_for_recovery_without_stale_after_unchanged_behavior( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + sid = uuid.uuid4() + await storage.create_saga(saga_id=sid, name="saga", context=test_context) + await storage.update_status(sid, SagaStatus.RUNNING) + ids = await storage.get_sagas_for_recovery(limit=10) + assert sid in ids + + async def test_get_sagas_for_recovery_filters_by_saga_name_when_provided( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id_foo1, id_foo2, id_bar = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + await storage.create_saga(saga_id=id_foo1, name="OrderSaga", context=test_context) + await storage.create_saga(saga_id=id_foo2, name="OrderSaga", context=test_context) + await storage.create_saga(saga_id=id_bar, name="PaymentSaga", context=test_context) + await storage.update_status(id_foo1, SagaStatus.RUNNING) + await storage.update_status(id_foo2, SagaStatus.RUNNING) + await storage.update_status(id_bar, SagaStatus.RUNNING) + ids_order = await storage.get_sagas_for_recovery(limit=10, saga_name="OrderSaga") + assert set(ids_order) == {id_foo1, id_foo2} + ids_payment = await storage.get_sagas_for_recovery(limit=10, saga_name="PaymentSaga") + assert ids_payment == [id_bar] + ids_none = await storage.get_sagas_for_recovery(limit=10, saga_name="NonExistentSaga") + assert ids_none == [] + + async def test_get_sagas_for_recovery_saga_name_none_returns_all_types( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id1, id2 = uuid.uuid4(), uuid.uuid4() + await storage.create_saga(saga_id=id1, name="SagaA", context=test_context) + await storage.create_saga(saga_id=id2, name="SagaB", context=test_context) + await storage.update_status(id1, SagaStatus.RUNNING) + await storage.update_status(id2, SagaStatus.RUNNING) + ids = await storage.get_sagas_for_recovery(limit=10, saga_name=None) + assert set(ids) == {id1, id2} + + async def test_get_sagas_for_recovery_empty_when_none_recoverable( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + sid = uuid.uuid4() + await storage.create_saga(saga_id=sid, name="saga", context=test_context) + await storage.update_status(sid, SagaStatus.COMPLETED) + ids = await storage.get_sagas_for_recovery(limit=10) + assert ids == [] + + async def test_get_sagas_for_recovery_excludes_pending_and_completed( + self, + storage: SqlAlchemySagaStorage, + test_context: dict[str, str], + ) -> None: + id_pending, id_completed = uuid.uuid4(), uuid.uuid4() + await storage.create_saga(saga_id=id_pending, name="saga", context=test_context) + await storage.create_saga(saga_id=id_completed, name="saga", context=test_context) + await storage.update_status(id_completed, SagaStatus.COMPLETED) + ids = await storage.get_sagas_for_recovery(limit=10) + assert id_pending not in ids + assert id_completed not in ids + + async def test_increment_recovery_attempts_increments_counter( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="saga", context=test_context) + await storage.update_status(saga_id, SagaStatus.RUNNING) + ids_before = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert saga_id in ids_before + for _ in range(5): + await storage.increment_recovery_attempts(saga_id) + ids_after = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert saga_id not in ids_after + + async def test_increment_recovery_attempts_with_new_status( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="saga", context=test_context) + await storage.update_status(saga_id, SagaStatus.RUNNING) + await storage.increment_recovery_attempts(saga_id, new_status=SagaStatus.FAILED) + status, _, _ = await storage.load_saga_state(saga_id) + assert status == SagaStatus.FAILED + + async def test_increment_recovery_attempts_raises_when_saga_not_found( + self, + storage: SqlAlchemySagaStorage, + ) -> None: + unknown_id = uuid.uuid4() + with pytest.raises(ValueError, match="not found"): + await storage.increment_recovery_attempts(unknown_id) + + async def test_set_recovery_attempts_sets_value( + self, + storage: SqlAlchemySagaStorage, + saga_id: uuid.UUID, + test_context: dict[str, str], + ) -> None: + await storage.create_saga(saga_id=saga_id, name="saga", context=test_context) + await storage.update_status(saga_id, SagaStatus.RUNNING) + await storage.increment_recovery_attempts(saga_id) + await storage.increment_recovery_attempts(saga_id) + await storage.set_recovery_attempts(saga_id, 0) + ids_reset = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert saga_id in ids_reset + await storage.set_recovery_attempts(saga_id, 5) + ids_max = await storage.get_sagas_for_recovery(limit=10, max_recovery_attempts=5) + assert saga_id not in ids_max + + async def test_set_recovery_attempts_raises_when_saga_not_found( + self, + storage: SqlAlchemySagaStorage, + ) -> None: + unknown_id = uuid.uuid4() + with pytest.raises(ValueError, match="not found"): + await storage.set_recovery_attempts(unknown_id, 0) diff --git a/tests/pytest-config.ini b/tests/pytest-config.ini index 0cd0432..fc3e329 100644 --- a/tests/pytest-config.ini +++ b/tests/pytest-config.ini @@ -3,5 +3,7 @@ asyncio_mode = auto norecursedirs = benchmarks env = DATABASE_DSN=mysql+asyncmy://cqrs:cqrs@localhost:3307/test_cqrs + DATABASE_DSN_MYSQL=mysql+asyncmy://cqrs:cqrs@localhost:3307/test_cqrs + DATABASE_DSN_POSTGRESQL=postgresql+asyncpg://cqrs:cqrs@localhost:5433/cqrs filterwarnings = ignore::DeprecationWarning:aio_pika.* diff --git a/tests/unit/test_saga/test_saga_basic.py b/tests/unit/test_saga/test_saga_basic.py index b5a9439..d06df57 100644 --- a/tests/unit/test_saga/test_saga_basic.py +++ b/tests/unit/test_saga/test_saga_basic.py @@ -313,4 +313,4 @@ async def test_saga_step_result_contains_correct_metadata( assert step_result.error_message is None assert step_result.error_traceback is None assert step_result.error_type is None - assert step_result.saga_id is not None \ No newline at end of file + assert step_result.saga_id is not None From 963ac2f771da36160d6eab19290f5be3bf395d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 13:53:34 +0300 Subject: [PATCH 07/17] Fix after pre-commit --- .pre-commit-config.yaml | 2 +- tests/benchmarks/__init__.py | 2 +- tests/benchmarks/conftest.py | 29 +++++++++++++++++-- .../dataclasses/test_benchmark_saga_memory.py | 2 +- .../test_benchmark_saga_sqlalchemy.py | 2 +- .../default/test_benchmark_saga_memory.py | 2 +- .../default/test_benchmark_saga_sqlalchemy.py | 2 +- tests/benchmarks/storage_legacy.py | 29 ------------------- 8 files changed, 33 insertions(+), 37 deletions(-) delete mode 100644 tests/benchmarks/storage_legacy.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ccda8e..67e2f15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: check-added-large-files - args: - --pytest-test-first - exclude: (^tests/mock/|^tests/integration/|^tests/fixtures) + exclude: (^tests/mock/|^tests/integration/|^tests/fixtures|conftest\.py$) id: name-tests-test - id: check-merge-conflict - id: check-json diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py index c038b0c..d6cb596 100644 --- a/tests/benchmarks/__init__.py +++ b/tests/benchmarks/__init__.py @@ -1 +1 @@ -# Benchmark package; shared helpers in storage_legacy.py +# Benchmark package; shared helpers in conftest.py diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index e450093..603269c 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -1,12 +1,37 @@ -"""Shared fixtures for benchmarks. Engine and loop are session-scoped for saga SQLAlchemy benchmarks.""" +"""Shared fixtures and legacy storage classes for benchmarks.""" + +from __future__ import annotations import asyncio +import contextlib import os import pytest from sqlalchemy.ext.asyncio import create_async_engine -from cqrs.saga.storage.sqlalchemy import Base +from cqrs.saga.storage.memory import MemorySagaStorage +from cqrs.saga.storage.protocol import SagaStorageRun +from cqrs.saga.storage.sqlalchemy import Base, SqlAlchemySagaStorage + + +class MemorySagaStorageLegacy(MemorySagaStorage): + """Memory storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """Raise NotImplementedError so benchmarks use the legacy commit-per-call path.""" + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") + + +class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): + """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" + + def create_run( + self, + ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: + """Raise NotImplementedError so benchmarks use the legacy commit-per-call path.""" + raise NotImplementedError("Legacy storage: create_run disabled for benchmark") @pytest.fixture(scope="session") diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index a92f399..92d34f1 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -16,7 +16,7 @@ from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.memory import MemorySagaStorage -from ..storage_legacy import MemorySagaStorageLegacy +from ..conftest import MemorySagaStorageLegacy @dataclasses.dataclass diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index 615519e..d06739b 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -10,7 +10,7 @@ from cqrs.saga.saga import Saga from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage -from ..storage_legacy import SqlAlchemySagaStorageLegacy +from ..conftest import SqlAlchemySagaStorageLegacy from .test_benchmark_saga_memory import ( OrderContext, ProcessPaymentStep, diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index e3aaefa..1f7856b 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -16,7 +16,7 @@ from cqrs.saga.step import SagaStepHandler, SagaStepResult from cqrs.saga.storage.memory import MemorySagaStorage -from ..storage_legacy import MemorySagaStorageLegacy +from ..conftest import MemorySagaStorageLegacy @dataclasses.dataclass diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index 34339c0..05b31d5 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -10,7 +10,7 @@ from cqrs.saga.saga import Saga from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage -from ..storage_legacy import SqlAlchemySagaStorageLegacy +from ..conftest import SqlAlchemySagaStorageLegacy from .test_benchmark_saga_memory import ( OrderContext, ProcessPaymentStep, diff --git a/tests/benchmarks/storage_legacy.py b/tests/benchmarks/storage_legacy.py deleted file mode 100644 index 2da7915..0000000 --- a/tests/benchmarks/storage_legacy.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Shared legacy storage classes for benchmark tests (no create_run; commit-per-call path).""" - -from __future__ import annotations - -import contextlib - -from cqrs.saga.storage.memory import MemorySagaStorage -from cqrs.saga.storage.protocol import SagaStorageRun -from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage - - -class MemorySagaStorageLegacy(MemorySagaStorage): - """Memory storage without create_run: forces legacy path (commit per call).""" - - def create_run( - self, - ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """Raise NotImplementedError so benchmarks use the legacy commit-per-call path.""" - raise NotImplementedError("Legacy storage: create_run disabled for benchmark") - - -class SqlAlchemySagaStorageLegacy(SqlAlchemySagaStorage): - """SQLAlchemy storage without create_run: forces legacy path (commit per call).""" - - def create_run( - self, - ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: - """Raise NotImplementedError so benchmarks use the legacy commit-per-call path.""" - raise NotImplementedError("Legacy storage: create_run disabled for benchmark") From 54bb87cebea3a82a098784677b80fa0424bf4da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:00:53 +0300 Subject: [PATCH 08/17] fix ruff format --- ..._injector_integration_practical_example.py | 4 +- examples/kafka_event_consuming.py | 4 +- examples/request_response_types.py | 4 +- examples/saga.py | 10 ++--- examples/saga_fallback.py | 10 ++--- examples/saga_recovery.py | 22 +++++----- src/cqrs/deserializers/json.py | 3 +- src/cqrs/outbox/mock.py | 4 +- src/cqrs/outbox/sqlalchemy.py | 12 ++---- src/cqrs/requests/map.py | 5 +-- src/cqrs/requests/mermaid.py | 20 +++------ src/cqrs/saga/execution.py | 12 +++--- src/cqrs/saga/mermaid.py | 32 ++++----------- src/cqrs/saga/recovery.py | 3 +- src/cqrs/saga/storage/memory.py | 41 ++++++++++--------- src/cqrs/saga/validation.py | 16 ++------ tests/integration/test_event_outbox.py | 4 +- tests/unit/test_cor_mermaid.py | 6 +-- tests/unit/test_deserializers.py | 4 +- .../test_saga/test_saga_compensation_retry.py | 4 +- tests/unit/test_saga/test_saga_storage_run.py | 26 ++++++------ tests/unit/test_saga/test_saga_to_mermaid.py | 40 ++++-------------- 22 files changed, 101 insertions(+), 185 deletions(-) diff --git a/examples/dependency_injector_integration_practical_example.py b/examples/dependency_injector_integration_practical_example.py index 8db017e..5c4e965 100644 --- a/examples/dependency_injector_integration_practical_example.py +++ b/examples/dependency_injector_integration_practical_example.py @@ -351,9 +351,7 @@ def setup_logging() -> None: ) # Add a StreamHandler if none exists - has_stream_handler = any( - isinstance(h, logging.StreamHandler) for h in root_logger.handlers - ) + has_stream_handler = any(isinstance(h, logging.StreamHandler) for h in root_logger.handlers) if not has_stream_handler: stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.DEBUG) diff --git a/examples/kafka_event_consuming.py b/examples/kafka_event_consuming.py index 989b033..b6f7a89 100644 --- a/examples/kafka_event_consuming.py +++ b/examples/kafka_event_consuming.py @@ -158,9 +158,7 @@ def mediator_factory() -> cqrs.EventMediator: decoder=empty_message_decoder, ) async def hello_world_event_handler( - body: cqrs.NotificationEvent[HelloWorldPayload] - | deserializers.DeserializeJsonError - | None, + body: cqrs.NotificationEvent[HelloWorldPayload] | deserializers.DeserializeJsonError | None, msg: kafka.KafkaMessage, mediator: cqrs.EventMediator = faststream.Depends(mediator_factory), ): diff --git a/examples/request_response_types.py b/examples/request_response_types.py index 55ff351..12b76b9 100644 --- a/examples/request_response_types.py +++ b/examples/request_response_types.py @@ -256,9 +256,7 @@ async def handle(self, request: GetUserQuery) -> UserDetailsResponse: raise ValueError(f"User {request.user_id} not found") user = USER_STORAGE[request.user_id] - total_orders = sum( - 1 for order in ORDER_STORAGE.values() if order["user_id"] == request.user_id - ) + total_orders = sum(1 for order in ORDER_STORAGE.values() if order["user_id"] == request.user_id) return UserDetailsResponse( user_id=user["user_id"], diff --git a/examples/saga.py b/examples/saga.py index a0710d6..417eb54 100644 --- a/examples/saga.py +++ b/examples/saga.py @@ -277,15 +277,15 @@ async def create_shipment( ) -> tuple[str, str]: """ Create a shipment for an order and record its tracking number. - + Parameters: order_id (str): Identifier of the order to ship. items (list[str]): List of item identifiers included in the shipment. address (str): Shipping address; must not be empty. - + Returns: tuple[str, str]: A tuple containing the created `shipment_id` and its `tracking_number`. - + Raises: ValueError: If `address` is empty. """ @@ -484,7 +484,7 @@ class OrderSaga(Saga[OrderContext]): async def run_successful_saga() -> None: """ Run an example order-processing saga and print the per-step progress and final results. - + Sets up mock services, dependency injection, and in-memory saga storage; executes the OrderSaga with a generated saga ID, prints each completed step, then prints the final saga status, context fields (inventory reservation, payment ID, shipment ID) and the persisted execution log. If saga execution fails, the failure is printed and the exception is re-raised. """ print("\n" + "=" * 70) @@ -753,4 +753,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/examples/saga_fallback.py b/examples/saga_fallback.py index dd95fa8..78a8fa9 100644 --- a/examples/saga_fallback.py +++ b/examples/saga_fallback.py @@ -146,12 +146,12 @@ async def act( ) -> SagaStepResult[OrderContext, ReserveInventoryResponse]: """ Simulate a failing primary reservation step for the saga. - + This action always raises a RuntimeError to emulate an unavailable downstream service and trigger fallback or compensation behavior. - + Parameters: context (OrderContext): Shared saga context containing order details (e.g., order_id, user_id, amount, reservation_id). - + Raises: RuntimeError: Indicates the primary step failed (service unavailable). """ @@ -283,7 +283,7 @@ async def run_saga( async def main() -> None: """ Run an interactive demonstration of the saga fallback pattern with a circuit breaker. - + Executes three scenarios that show a failing primary step with an automatic fallback, the circuit breaker opening after a configurable number of failures, and fail-fast behavior when the circuit is open. Also conditionally demonstrates configuring a Redis-backed circuit breaker storage, prints per-scenario results and a summary, and informs about missing optional dependencies. """ print("\n" + "=" * 80) @@ -433,4 +433,4 @@ class OrderSagaWithRedisBreaker(Saga[OrderContext]): if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/examples/saga_recovery.py b/examples/saga_recovery.py index 3400c94..a7848be 100644 --- a/examples/saga_recovery.py +++ b/examples/saga_recovery.py @@ -283,15 +283,15 @@ async def create_shipment( ) -> tuple[str, str]: """ Create a shipment record for the given order and generate a tracking number. - + Parameters: order_id (str): Identifier of the order. items (list[str]): Items included in the shipment. address (str): Destination shipping address. - + Returns: tuple[str, str]: A tuple containing the shipment ID and the tracking number. - + Raises: ValueError: If `address` is empty. """ @@ -533,7 +533,7 @@ async def resolve(self, type_: type) -> typing.Any: async def simulate_interrupted_saga() -> tuple[uuid.UUID, MemorySagaStorage]: """ Simulate a saga that is interrupted after the inventory reservation step to produce a recoverable persisted state. - + Returns: tuple: saga_id (uuid.UUID): Identifier of the created saga. @@ -632,9 +632,9 @@ async def recover_interrupted_saga( ) -> None: """ Recover and complete an interrupted saga using persisted state. - + Loads the saga state from storage, reconstructs the saga context, resumes execution from the last completed step, and completes any remaining steps to restore eventual consistency. - + Parameters: saga_id (uuid.UUID): Identifier of the saga instance to recover. storage (MemorySagaStorage): Durable storage containing the saga's persisted state and step history. @@ -698,9 +698,9 @@ async def recover_interrupted_saga( async def simulate_interrupted_compensation() -> tuple[uuid.UUID, MemorySagaStorage]: """ Simulate a saga that fails and is interrupted during compensation. - + Sets up services, a saga, and a failing shipment step to trigger compensation that is then artificially interrupted; returns identifiers and storage state for performing recovery in a separate run. - + Returns: tuple[uuid.UUID, MemorySagaStorage]: The saga ID and the in-memory storage containing the persisted saga state and step history after the simulated interruption. """ @@ -813,9 +813,9 @@ async def recover_interrupted_compensation( ) -> None: """ Recover and complete an interrupted compensation for a saga. - + Loads the saga state from the provided storage using the given saga identifier and drives any incomplete compensation steps to completion, ensuring resources (inventory, payments, shipments) are released and the system reaches a consistent state. Progress and final status are printed to stdout. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to recover. storage (MemorySagaStorage): Persistent storage containing the saga state and step history. @@ -924,4 +924,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/src/cqrs/deserializers/json.py b/src/cqrs/deserializers/json.py index 765b8cb..9f25565 100644 --- a/src/cqrs/deserializers/json.py +++ b/src/cqrs/deserializers/json.py @@ -75,8 +75,7 @@ def __init__(self, model: typing.Type[typing.Any]): getattr(model, "from_dict", None), ): raise TypeError( - f"Model {model} does not implement Deserializable protocol: " - "missing 'from_dict' classmethod", + f"Model {model} does not implement Deserializable protocol: " "missing 'from_dict' classmethod", ) # Store model - type is preserved through generic parameter _T for return type self._model: typing.Type[typing.Any] = model diff --git a/src/cqrs/outbox/mock.py b/src/cqrs/outbox/mock.py index b2fec1c..6375d27 100644 --- a/src/cqrs/outbox/mock.py +++ b/src/cqrs/outbox/mock.py @@ -31,9 +31,7 @@ async def get_many( topic: typing.Text | None = None, ) -> typing.List[repository.OutboxedEvent]: return list( - filter(lambda e: topic == e.topic, self.session.values()) - if topic - else list(self.session.values()), + filter(lambda e: topic == e.topic, self.session.values()) if topic else list(self.session.values()), ) async def update_status( diff --git a/src/cqrs/outbox/sqlalchemy.py b/src/cqrs/outbox/sqlalchemy.py index 1e533f7..b4f8bc4 100644 --- a/src/cqrs/outbox/sqlalchemy.py +++ b/src/cqrs/outbox/sqlalchemy.py @@ -89,9 +89,7 @@ class OutboxModel(Base): ) def row_to_dict(self) -> typing.Dict[typing.Text, typing.Any]: - return { - column.name: getattr(self, column.name) for column in self.__table__.columns - } + return {column.name: getattr(self, column.name) for column in self.__table__.columns} @classmethod def get_batch_query( @@ -132,9 +130,7 @@ def update_status_query( if status == repository.EventStatus.NOT_PRODUCED: values["flush_counter"] += 1 - return ( - sqlalchemy.update(cls).where(cls.id == outboxed_event_id).values(**values) - ) + return sqlalchemy.update(cls).where(cls.id == outboxed_event_id).values(**values) @classmethod def status_sorting_case(cls) -> sqlalchemy.Case: @@ -212,9 +208,7 @@ async def get_many( topic: typing.Text | None = None, ) -> typing.List[repository.OutboxedEvent]: events: typing.Sequence[OutboxModel] = ( - (await self.session.execute(OutboxModel.get_batch_query(batch_size, topic))) - .scalars() - .all() + (await self.session.execute(OutboxModel.get_batch_query(batch_size, topic))).scalars().all() ) result = [] diff --git a/src/cqrs/requests/map.py b/src/cqrs/requests/map.py index faea78c..8a368c1 100644 --- a/src/cqrs/requests/map.py +++ b/src/cqrs/requests/map.py @@ -12,10 +12,7 @@ _KT = typing.TypeVar("_KT", bound=typing.Type[IRequest]) # Type alias for handler types that can be bound to requests -HandlerType = ( - typing.Type[RequestHandler | StreamingRequestHandler] - | typing.List[typing.Type[CORRequestHandler]] -) +HandlerType = typing.Type[RequestHandler | StreamingRequestHandler] | typing.List[typing.Type[CORRequestHandler]] class RequestMap(typing.Dict[_KT, HandlerType]): diff --git a/src/cqrs/requests/mermaid.py b/src/cqrs/requests/mermaid.py index d3e6d13..eb87a90 100644 --- a/src/cqrs/requests/mermaid.py +++ b/src/cqrs/requests/mermaid.py @@ -60,9 +60,7 @@ def sequence(self) -> str: alias = f"H{idx}" handler_aliases[handler_name] = alias # Truncate long handler names for better diagram readability - display_name = ( - handler_name if len(handler_name) <= 30 else handler_name[:27] + "..." - ) + display_name = handler_name if len(handler_name) <= 30 else handler_name[:27] + "..." participants.append(f"{alias} as {display_name}") lines = ["sequenceDiagram"] @@ -214,9 +212,7 @@ def class_diagram(self) -> str: fields = request_type.__dataclass_fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type.__name__ - if hasattr(field_info.type, "__name__") - else str(field_info.type) + field_info.type.__name__ if hasattr(field_info.type, "__name__") else str(field_info.type) ) lines.append(f" +{field_name}: {field_type}") elif hasattr(request_type, "model_fields"): # Pydantic v2 @@ -232,9 +228,7 @@ def class_diagram(self) -> str: fields = request_type.__fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type_.__name__ - if hasattr(field_info.type_, "__name__") - else str(field_info.type_) + field_info.type_.__name__ if hasattr(field_info.type_, "__name__") else str(field_info.type_) ) lines.append(f" +{field_name}: {field_type}") lines.append(" }") @@ -249,9 +243,7 @@ def class_diagram(self) -> str: fields = response_type.__dataclass_fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type.__name__ - if hasattr(field_info.type, "__name__") - else str(field_info.type) + field_info.type.__name__ if hasattr(field_info.type, "__name__") else str(field_info.type) ) lines.append(f" +{field_name}: {field_type}") elif hasattr(response_type, "model_fields"): # Pydantic v2 @@ -267,9 +259,7 @@ def class_diagram(self) -> str: fields = response_type.__fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type_.__name__ - if hasattr(field_info.type_, "__name__") - else str(field_info.type_) + field_info.type_.__name__ if hasattr(field_info.type_, "__name__") else str(field_info.type_) ) lines.append(f" +{field_name}: {field_type}") lines.append(" }") diff --git a/src/cqrs/saga/execution.py b/src/cqrs/saga/execution.py index 84a63bf..8b6e1d0 100644 --- a/src/cqrs/saga/execution.py +++ b/src/cqrs/saga/execution.py @@ -25,7 +25,7 @@ def __init__( ) -> None: """ Create a SagaStateManager bound to a specific saga identifier and storage backend. - + Parameters: saga_id: Identifier for the saga instance. storage: Storage backend implementing ISagaStorage or SagaStorageRun used to persist saga state and history. @@ -85,7 +85,7 @@ def __init__( ) -> None: """ Construct a SagaRecoveryManager that holds the identifiers, storage, DI container, and configured saga steps required to reconstruct a saga's execution state. - + Parameters: saga_id: Identifier for the saga instance (e.g., UUID or other unique value). storage: Persistence backend implementing saga history operations (ISagaStorage or SagaStorageRun). @@ -100,7 +100,7 @@ def __init__( async def load_completed_step_names(self) -> set[str]: """ Return the names of saga steps that completed their primary ("act") action. - + Returns: set[str]: Step names recorded with status `SagaStepStatus.COMPLETED` and action `"act"`. """ @@ -113,10 +113,10 @@ async def reconstruct_completed_steps( ) -> list[SagaStepHandler[SagaContext, typing.Any]]: """ Reconstructs and returns the resolved step handler instances corresponding to the completed steps, preserving saga execution order. - + Parameters: completed_step_names (set[str]): Names of steps that completed the "act" action. - + Returns: list[SagaStepHandler[SagaContext, typing.Any]]: Resolved step handler instances in execution order. For Fallback wrappers, the primary handler is chosen if its name appears in completed_step_names; otherwise the fallback handler is chosen when present. """ @@ -365,4 +365,4 @@ async def execute_fallback_step( raise fallback_error else: # Should not fallback, re-raise original error - raise primary_error \ No newline at end of file + raise primary_error diff --git a/src/cqrs/saga/mermaid.py b/src/cqrs/saga/mermaid.py index 719ed0f..a2e745a 100644 --- a/src/cqrs/saga/mermaid.py +++ b/src/cqrs/saga/mermaid.py @@ -72,16 +72,8 @@ def sequence(self) -> str: fallback_aliases[fallback_name] = fallback_alias # Truncate long names - primary_display = ( - primary_name - if len(primary_name) <= 30 - else primary_name[:27] + "..." - ) - fallback_display = ( - fallback_name - if len(fallback_name) <= 30 - else fallback_name[:27] + "..." - ) + primary_display = primary_name if len(primary_name) <= 30 else primary_name[:27] + "..." + fallback_display = fallback_name if len(fallback_name) <= 30 else fallback_name[:27] + "..." participants.append(f"{primary_alias} as {primary_display}") participants.append( @@ -93,9 +85,7 @@ def sequence(self) -> str: step_name = step_item.__name__ alias = f"S{step_idx}" step_aliases[step_name] = alias - display_name = ( - step_name if len(step_name) <= 30 else step_name[:27] + "..." - ) + display_name = step_name if len(step_name) <= 30 else step_name[:27] + "..." participants.append(f"{alias} as {display_name}") step_idx += 1 @@ -221,9 +211,7 @@ def class_diagram(self) -> str: steps = self._saga.steps if not steps: - return ( - "classDiagram\n class Saga\n Note for Saga: No steps configured" - ) + return "classDiagram\n class Saga\n Note for Saga: No steps configured" # Collect all types context_types: set[type] = set() @@ -363,9 +351,7 @@ def class_diagram(self) -> str: fields = context_type.__dataclass_fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type.__name__ - if hasattr(field_info.type, "__name__") - else str(field_info.type) + field_info.type.__name__ if hasattr(field_info.type, "__name__") else str(field_info.type) ) lines.append(f" +{field_name}: {field_type}") lines.append(" }") @@ -380,9 +366,7 @@ def class_diagram(self) -> str: fields = response_type.__dataclass_fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type.__name__ - if hasattr(field_info.type, "__name__") - else str(field_info.type) + field_info.type.__name__ if hasattr(field_info.type, "__name__") else str(field_info.type) ) lines.append(f" +{field_name}: {field_type}") elif hasattr(response_type, "model_fields"): # Pydantic v2 @@ -398,9 +382,7 @@ def class_diagram(self) -> str: fields = response_type.__fields__ for field_name, field_info in fields.items(): field_type = ( - field_info.type_.__name__ - if hasattr(field_info.type_, "__name__") - else str(field_info.type_) + field_info.type_.__name__ if hasattr(field_info.type_, "__name__") else str(field_info.type_) ) lines.append(f" +{field_name}: {field_type}") lines.append(" }") diff --git a/src/cqrs/saga/recovery.py b/src/cqrs/saga/recovery.py index 5d80dc4..558c30c 100644 --- a/src/cqrs/saga/recovery.py +++ b/src/cqrs/saga/recovery.py @@ -109,8 +109,7 @@ async def recover_saga( error_msg = str(e) if "recovered in" in error_msg and "state" in error_msg: logger.warning( - f"Saga {saga_id} recovery completed compensation. " - "Forward execution was not allowed.", + f"Saga {saga_id} recovery completed compensation. " "Forward execution was not allowed.", ) # Re-raise to allow callers to handle this case raise diff --git a/src/cqrs/saga/storage/memory.py b/src/cqrs/saga/storage/memory.py index a1ec80f..504f4a6 100644 --- a/src/cqrs/saga/storage/memory.py +++ b/src/cqrs/saga/storage/memory.py @@ -18,7 +18,7 @@ class _MemorySagaStorageRun(SagaStorageRun): def __init__(self, storage: "MemorySagaStorage") -> None: """ Initialize the run and bind it to the provided MemorySagaStorage. - + Parameters: storage (MemorySagaStorage): Underlying in-memory storage instance used to delegate saga operations. """ @@ -32,12 +32,12 @@ async def create_saga( ) -> None: """ Create a new saga entry in the underlying memory storage. - + Parameters: saga_id (uuid.UUID): Unique identifier for the saga. name (str): Human-readable saga name. context (dict[str, typing.Any]): Initial saga context payload. - + Raises: ValueError: If a saga with the same `saga_id` already exists. """ @@ -51,12 +51,12 @@ async def update_context( ) -> None: """ Update the stored context for the given saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose context will be updated. context (dict[str, typing.Any]): New context to store for the saga. current_version (int | None): If provided, require the stored saga version to match this value (optimistic locking). - + Raises: ValueError: If the saga_id does not exist. SagaConcurrencyError: If current_version is provided and does not match the stored version. @@ -70,11 +70,11 @@ async def update_status( ) -> None: """ Update the stored status of the saga identified by `saga_id`. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to update. status (SagaStatus): New status to set for the saga. - + Raises: ValueError: If no saga exists with the given `saga_id`. """ @@ -90,7 +90,7 @@ async def log_step( ) -> None: """ Log a step entry for the given saga into the underlying storage. - + Parameters: saga_id (uuid.UUID): Identifier of the saga. step_name (str): Name of the saga step. @@ -114,11 +114,11 @@ async def load_saga_state( ) -> tuple[SagaStatus, dict[str, typing.Any], int]: """ Load the current state for a saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to load. read_for_update (bool): If True, acquire the state for update (may be used for optimistic locking or exclusive access). - + Returns: tuple[SagaStatus, dict[str, typing.Any], int]: A tuple containing the saga's status, its context dictionary, and the current version number. """ @@ -133,10 +133,10 @@ async def get_step_history( ) -> list[SagaLogEntry]: """ Retrieve the step log/history for a saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose step history is requested. - + Returns: list[SagaLogEntry]: Saga step log entries sorted by timestamp in ascending order (oldest first). Returns an empty list if no logs exist. """ @@ -145,7 +145,7 @@ async def get_step_history( async def commit(self) -> None: """ No-op commit for an in-memory saga run; provided to satisfy the SagaStorageRun interface. - + This method intentionally performs no action because the memory storage does not require an explicit commit. """ pass @@ -164,7 +164,7 @@ def __init__(self) -> None: # Structure: {saga_id: {name, status, context, created_at, updated_at, version}} """ Initialize in-memory storage for sagas and their step logs. - + Creates two internal mappings: - _sagas: maps saga_id (UUID) to a dictionary containing keys `name`, `status`, `context`, `created_at`, `updated_at`, and `version`. - _logs: maps saga_id (UUID) to a list of SagaLogEntry objects representing the saga's step history. @@ -178,10 +178,11 @@ def create_run( ) -> contextlib.AbstractAsyncContextManager[SagaStorageRun]: """ Provide an asynchronous context manager that yields a SagaStorageRun bound to this storage. - + Returns: An asynchronous context manager that yields a `SagaStorageRun` instance backed by this `MemorySagaStorage`. """ + @contextlib.asynccontextmanager async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: yield _MemorySagaStorageRun(self) @@ -196,12 +197,12 @@ async def create_saga( ) -> None: """ Create a new saga record in the in-memory store. - + Parameters: saga_id (uuid.UUID): Identifier for the saga; must not already exist. name (str): Human-readable name for the saga. context (dict[str, typing.Any]): Initial context payload for the saga. - + Raises: ValueError: If a saga with `saga_id` already exists. """ @@ -305,13 +306,13 @@ async def get_sagas_for_recovery( ) -> list[uuid.UUID]: """ Selects saga IDs eligible for recovery based on status, recovery attempts, staleness, and an optional name filter. - + Parameters: limit (int): Maximum number of saga IDs to return. max_recovery_attempts (int): Upper bound (exclusive) on recovery attempts; only sagas with fewer attempts are considered. stale_after_seconds (int | None): If provided, only sagas last updated earlier than this many seconds before now are considered; if None, staleness is ignored. saga_name (str | None): If provided, only sagas with this name are considered; if None, name is not filtered. - + Returns: list[uuid.UUID]: Up to `limit` saga IDs sorted by oldest `updated_at` first that match the recovery criteria. """ @@ -353,4 +354,4 @@ async def set_recovery_attempts( data = self._sagas[saga_id] data["recovery_attempts"] = attempts data["updated_at"] = datetime.datetime.now(datetime.timezone.utc) - data["version"] += 1 \ No newline at end of file + data["version"] += 1 diff --git a/src/cqrs/saga/validation.py b/src/cqrs/saga/validation.py index e304b22..688af2f 100644 --- a/src/cqrs/saga/validation.py +++ b/src/cqrs/saga/validation.py @@ -30,10 +30,7 @@ def extract_from_class(klass: type, saga_base_class: type) -> type | None: if hasattr(klass, "__orig_bases__"): for base in klass.__orig_bases__: # type: ignore[attr-defined] # Check if this is a GenericAlias for Saga - if ( - isinstance(base, types.GenericAlias) - and base.__origin__ is saga_base_class - ): # type: ignore[attr-defined] + if isinstance(base, types.GenericAlias) and base.__origin__ is saga_base_class: # type: ignore[attr-defined] args = typing.get_args(base) if args: return args[0] # type: ignore[return-value] @@ -65,10 +62,7 @@ def extract_from_step(step_type: type) -> type | None: if hasattr(step_type, "__orig_bases__"): for base in step_type.__orig_bases__: # type: ignore[attr-defined] # Check if this is a GenericAlias for SagaStepHandler - if ( - isinstance(base, types.GenericAlias) - and base.__origin__ is SagaStepHandler - ): # type: ignore[attr-defined] + if isinstance(base, types.GenericAlias) and base.__origin__ is SagaStepHandler: # type: ignore[attr-defined] args = typing.get_args(base) if args: return args[0] @@ -202,8 +196,7 @@ def validate_steps( """ if not isinstance(steps, list): raise TypeError( - f"{self._saga_name} steps must be a list of step handler types, " - f"got {type(steps).__name__}", + f"{self._saga_name} steps must be a list of step handler types, " f"got {type(steps).__name__}", ) if not steps: @@ -284,8 +277,7 @@ def _validate_regular_step( # Check if step is a subclass of SagaStepHandler if not issubclass(step_item, SagaStepHandler): raise TypeError( - f"{self._saga_name} steps[{index}] ({step_item.__name__}) " - "must be a subclass of SagaStepHandler", + f"{self._saga_name} steps[{index}] ({step_item.__name__}) " "must be a subclass of SagaStepHandler", ) # Validate context type compatibility diff --git a/tests/integration/test_event_outbox.py b/tests/integration/test_event_outbox.py index 648df38..40b11e7 100644 --- a/tests/integration/test_event_outbox.py +++ b/tests/integration/test_event_outbox.py @@ -61,9 +61,7 @@ async def test_outbox_add_3_event_positive(self, session): request = OutboxRequest(message="test_outbox_add_3_event_positive", count=3) await OutboxRequestHandler(repository).handle(request) - not_produced_events: typing.List[ - outbox_repository.OutboxedEvent - ] = await repository.get_many(3) + not_produced_events: typing.List[outbox_repository.OutboxedEvent] = await repository.get_many(3) await session.commit() assert len(not_produced_events) == 3 diff --git a/tests/unit/test_cor_mermaid.py b/tests/unit/test_cor_mermaid.py index 3fa2ff3..2906629 100644 --- a/tests/unit/test_cor_mermaid.py +++ b/tests/unit/test_cor_mermaid.py @@ -224,8 +224,7 @@ def test_class_diagram_relationships() -> None: # Check chain relationships (set_next) assert ( - "CreditCardHandler --> PayPalHandler" in diagram - or "CreditCardHandler --> PayPalHandler : set_next" in diagram + "CreditCardHandler --> PayPalHandler" in diagram or "CreditCardHandler --> PayPalHandler : set_next" in diagram ) assert ( "PayPalHandler --> BankTransferHandler" in diagram @@ -240,6 +239,5 @@ def test_class_diagram_relationships() -> None: # Check Handler to Response relationships assert ( - "CreditCardHandler ..> PaymentResult" in diagram - or "CreditCardHandler ..> PaymentResult : returns" in diagram + "CreditCardHandler ..> PaymentResult" in diagram or "CreditCardHandler ..> PaymentResult : returns" in diagram ) diff --git a/tests/unit/test_deserializers.py b/tests/unit/test_deserializers.py index 119f927..3421e30 100644 --- a/tests/unit/test_deserializers.py +++ b/tests/unit/test_deserializers.py @@ -86,9 +86,7 @@ def test_json_deserializer_missing_required_fields_negative(): # JSON with payload that has wrong type for required field 'bar' (string instead of int) # This should cause a validation error when Pydantic tries to validate the payload - incomplete_json = ( - '{"event_name": "test", "payload": {"foo": "bar", "bar": "not_an_int"}}' - ) + incomplete_json = '{"event_name": "test", "payload": {"foo": "bar", "bar": "not_an_int"}}' result = deserializer(incomplete_json) assert isinstance(result, json.DeserializeJsonError) diff --git a/tests/unit/test_saga/test_saga_compensation_retry.py b/tests/unit/test_saga/test_saga_compensation_retry.py index 6b05ce4..0fd3163 100644 --- a/tests/unit/test_saga/test_saga_compensation_retry.py +++ b/tests/unit/test_saga/test_saga_compensation_retry.py @@ -135,9 +135,7 @@ async def mock_sleep(delay: float) -> None: # Attempt 4 succeeds -> no wait expected_delays = [initial_delay * (backoff_multiplier**i) for i in range(3)] - assert ( - len(sleep_times) == 3 - ), f"Expected 3 sleep calls, got {len(sleep_times)}: {sleep_times}" + assert len(sleep_times) == 3, f"Expected 3 sleep calls, got {len(sleep_times)}: {sleep_times}" for actual, expected in zip(sleep_times, expected_delays): assert abs(actual - expected) < 0.01, f"Expected {expected}, got {actual}" diff --git a/tests/unit/test_saga/test_saga_storage_run.py b/tests/unit/test_saga/test_saga_storage_run.py index 2e0c575..d25ebda 100644 --- a/tests/unit/test_saga/test_saga_storage_run.py +++ b/tests/unit/test_saga/test_saga_storage_run.py @@ -30,7 +30,7 @@ def __init__(self) -> None: async def create_saga(self, saga_id: uuid.UUID, name: str, context: dict) -> None: """ Create a new saga record with the given identifier, name, and initial context. - + Parameters: saga_id (uuid.UUID): Unique identifier for the saga. name (str): Human-readable name of the saga. @@ -46,7 +46,7 @@ async def update_context( ) -> None: """ Update the stored context for a saga, optionally validating the expected current version. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose context will be updated. context (dict): New context data to persist for the saga. @@ -57,7 +57,7 @@ async def update_context( async def update_status(self, saga_id: uuid.UUID, status: SagaStatus) -> None: """ Update the stored status of a saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to update. status (SagaStatus): New status to set for the saga. @@ -74,7 +74,7 @@ async def log_step( ) -> None: """ Record the execution or compensation outcome of a saga step. - + Parameters: saga_id (uuid.UUID): Identifier of the saga. step_name (str): Name of the step being logged. @@ -92,11 +92,11 @@ async def load_saga_state( ) -> tuple[SagaStatus, dict, int]: """ Load the current state for a saga from the underlying storage. - + Parameters: saga_id (uuid.UUID): Identifier of the saga to load. read_for_update (bool): If True, load the state with intent to update (may acquire locks or use a read-for-update strategy). - + Returns: tuple[SagaStatus, dict, int]: A tuple containing the saga's status, its context dictionary, and the current version number. """ @@ -108,10 +108,10 @@ async def load_saga_state( async def get_step_history(self, saga_id: uuid.UUID) -> list: """ Return the step execution history for the given saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose history to retrieve. - + Returns: list: Step history records in chronological order. Each record describes the step name, action ("act" or "compensate"), step status, timestamp, and any optional details. """ @@ -126,13 +126,13 @@ async def get_sagas_for_recovery( ) -> list[uuid.UUID]: """ Selects saga IDs that are eligible for recovery. - + Parameters: limit (int): Maximum number of saga IDs to return. max_recovery_attempts (int): Only include sagas with fewer than this many recovery attempts. stale_after_seconds (int | None): If provided, only include sagas last updated more than this many seconds ago; if None, do not filter by staleness. saga_name (str | None): If provided, restrict results to sagas with this name. - + Returns: list[uuid.UUID]: Saga UUIDs that match the recovery criteria, up to `limit`. """ @@ -150,7 +150,7 @@ async def increment_recovery_attempts( ) -> None: """ Increment the recovery-attempts counter for a saga and optionally update its status. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose recovery attempts should be incremented. new_status (SagaStatus | None): If provided, update the saga's status to this value after incrementing attempts; otherwise leave status unchanged. @@ -160,7 +160,7 @@ async def increment_recovery_attempts( async def set_recovery_attempts(self, saga_id: uuid.UUID, attempts: int) -> None: """ Set the number of recovery attempts recorded for a saga. - + Parameters: saga_id (uuid.UUID): Identifier of the saga whose recovery attempts will be set. attempts (int): Number of recovery attempts to record; should be zero or a positive integer. @@ -289,4 +289,4 @@ async def test_storage_create_run_raises_not_implemented_by_default() -> None: """Default create_run() on a minimal storage raises NotImplementedError.""" storage = StorageWithoutCreateRun() with pytest.raises(NotImplementedError, match="does not support create_run"): - storage.create_run() \ No newline at end of file + storage.create_run() diff --git a/tests/unit/test_saga/test_saga_to_mermaid.py b/tests/unit/test_saga/test_saga_to_mermaid.py index 42afd39..d8f8bab 100644 --- a/tests/unit/test_saga/test_saga_to_mermaid.py +++ b/tests/unit/test_saga/test_saga_to_mermaid.py @@ -230,9 +230,7 @@ class TestSaga(Saga[OrderContext]): # Check that the name is truncated (should be max 30 chars + "...") assert "participant S1 as" in diagram # The full name should not appear, but truncated version should - participant_line = [ - line for line in diagram.split("\n") if "participant S1" in line - ][0] + participant_line = [line for line in diagram.split("\n") if "participant S1" in line][0] # Name should be truncated to 30 chars max assert len(participant_line.split("as")[1].strip()) <= 33 # 30 + "..." @@ -329,23 +327,13 @@ class TestSaga(Saga[OrderContext]): diagram = generator.class_diagram() # Check Saga to Step relationships - assert ( - "Saga --> ReserveInventoryStep" in diagram - or "Saga --> ReserveInventoryStep : contains" in diagram - ) - assert ( - "Saga --> ProcessPaymentStep" in diagram - or "Saga --> ProcessPaymentStep : contains" in diagram - ) - assert ( - "Saga --> ShipOrderStep" in diagram - or "Saga --> ShipOrderStep : contains" in diagram - ) + assert "Saga --> ReserveInventoryStep" in diagram or "Saga --> ReserveInventoryStep : contains" in diagram + assert "Saga --> ProcessPaymentStep" in diagram or "Saga --> ProcessPaymentStep : contains" in diagram + assert "Saga --> ShipOrderStep" in diagram or "Saga --> ShipOrderStep : contains" in diagram # Check Step to Context relationships assert ( - "ReserveInventoryStep ..> OrderContext" in diagram - or "ReserveInventoryStep ..> OrderContext : uses" in diagram + "ReserveInventoryStep ..> OrderContext" in diagram or "ReserveInventoryStep ..> OrderContext : uses" in diagram ) # Check Step to Response relationships @@ -358,8 +346,7 @@ class TestSaga(Saga[OrderContext]): or "ProcessPaymentStep ..> ProcessPaymentResponse : returns" in diagram ) assert ( - "ShipOrderStep ..> ShipOrderResponse" in diagram - or "ShipOrderStep ..> ShipOrderResponse : returns" in diagram + "ShipOrderStep ..> ShipOrderResponse" in diagram or "ShipOrderStep ..> ShipOrderResponse : returns" in diagram ) @@ -479,14 +466,8 @@ class TestSaga(Saga[OrderContext]): assert "class ReserveInventoryResponse" in diagram # Check relationships - assert ( - "Saga --> ReserveInventoryStep" in diagram - or "Saga --> ReserveInventoryStep : contains" in diagram - ) - assert ( - "Saga --> FallbackStep" in diagram - or "Saga --> FallbackStep : contains" in diagram - ) + assert "Saga --> ReserveInventoryStep" in diagram or "Saga --> ReserveInventoryStep : contains" in diagram + assert "Saga --> FallbackStep" in diagram or "Saga --> FallbackStep : contains" in diagram def test_sequence_diagram_fallback_single_step(saga_container: SagaContainer) -> None: @@ -540,7 +521,4 @@ class TestSaga(Saga[OrderContext]): if failure_section_start != -1: failure_section = diagram[failure_section_start:] # Should show primary failing, then fallback succeeding - assert ( - "Fallback triggered" in failure_section - or "fallback" in failure_section.lower() - ) + assert "Fallback triggered" in failure_section or "fallback" in failure_section.lower() From d233e0487ca575d1ecf27423f6ab62812b5ba60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:12:03 +0300 Subject: [PATCH 09/17] fix after review --- docker-compose-dev.yml | 2 +- docker-compose-test.yml | 2 +- examples/saga_recovery_scheduler.py | 16 ++- src/cqrs/saga/compensation.py | 2 +- src/cqrs/saga/saga.py | 2 + src/cqrs/saga/storage/sqlalchemy.py | 11 +- .../dataclasses/test_benchmark_saga_memory.py | 4 +- .../test_benchmark_saga_sqlalchemy.py | 4 +- .../default/test_benchmark_saga_memory.py | 9 +- .../default/test_benchmark_saga_sqlalchemy.py | 55 ++++---- tests/integration/conftest.py | 119 ++++++++++++++++++ .../test_saga_mediator_sqlalchemy_mysql.py | 101 --------------- .../test_saga_mediator_sqlalchemy_postgres.py | 101 --------------- .../test_saga_storage_sqlalchemy_mysql.py | 2 +- .../test_saga_storage_sqlalchemy_postgres.py | 16 ++- 15 files changed, 190 insertions(+), 256 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 064b5b1..fdb209c 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -16,7 +16,7 @@ services: volumes: - ./tests/init_database.sql:/data/application/init.sql postgres_dev: - image: postgres:latest + image: postgres:15.4 hostname: postgres-dev restart: always environment: diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 4e97ab1..60394a4 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -16,7 +16,7 @@ services: volumes: - ./tests/init_database.sql:/data/application/init.sql postgres_tests: - image: postgres:latest + image: postgres:15.4 hostname: postgres-test restart: always environment: diff --git a/examples/saga_recovery_scheduler.py b/examples/saga_recovery_scheduler.py index d838fe5..16d6d5e 100644 --- a/examples/saga_recovery_scheduler.py +++ b/examples/saga_recovery_scheduler.py @@ -79,7 +79,6 @@ import dataclasses import datetime import logging -import traceback import typing import uuid @@ -264,7 +263,12 @@ async def create_shipment( shipment_id = f"shipment_{order_id}" tracking_number = f"TRACK{self._tracking_counter:08d}" self._shipments[shipment_id] = tracking_number - logger.info(f" ✓ Created shipment {shipment_id} for order {order_id} (tracking: {tracking_number})") + logger.info( + " ✓ Created shipment %s for order %s (tracking: %s)", + shipment_id, + order_id, + tracking_number, + ) return shipment_id, tracking_number async def cancel_shipment(self, shipment_id: str) -> None: @@ -517,13 +521,13 @@ async def run_recovery_iteration( processed += 1 except RuntimeError as e: if "recovered in" in str(e) and "state" in str(e): - logger.info(f"Saga {saga_id} recovery completed compensation: {traceback.format_exc()}") + logger.info("Saga %s recovery completed compensation", saga_id) processed += 1 else: - logger.exception(f"Saga {saga_id} recovery failed: {traceback.format_exc()}") + logger.exception("Saga %s recovery failed", saga_id) processed += 1 except Exception: - logger.exception(f"Saga {saga_id} recovery failed: {traceback.format_exc()}") + logger.exception("Saga %s recovery failed", saga_id) processed += 1 return processed @@ -562,7 +566,7 @@ async def recovery_loop( logger.info("Recovery loop cancelled.") raise except Exception: - logger.exception(f"Recovery iteration failed: {traceback.format_exc()}") + logger.exception("Recovery iteration failed") if max_iterations is not None and iteration >= max_iterations: logger.info(f"Reached max_iterations={max_iterations}, stopping.") diff --git a/src/cqrs/saga/compensation.py b/src/cqrs/saga/compensation.py index 699ee82..5ab3eb5 100644 --- a/src/cqrs/saga/compensation.py +++ b/src/cqrs/saga/compensation.py @@ -141,7 +141,7 @@ async def compensate_steps( # Mark as failed eventually await self._storage.update_status(self._saga_id, SagaStatus.FAILED) else: - # If all compensations succeeded (or were skipped), mark as failed + # All compensations completed or were skipped — mark saga as FAILED because the original forward transaction failed await self._storage.update_status(self._saga_id, SagaStatus.FAILED) async def _compensate_step_with_retry( diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index db0709a..db5a3b1 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -386,6 +386,8 @@ async def _execute( SagaStepStatus.FAILED, str(e), ) + if run is not None: + await run.commit() self._error = e self._compensated = True await compensator.compensate_steps(self._completed_steps) diff --git a/src/cqrs/saga/storage/sqlalchemy.py b/src/cqrs/saga/storage/sqlalchemy.py index f892757..a1a4ae7 100644 --- a/src/cqrs/saga/storage/sqlalchemy.py +++ b/src/cqrs/saga/storage/sqlalchemy.py @@ -226,8 +226,11 @@ async def update_status( Note: The update is executed in the active database session; a commit is required to persist the change. + + Raises: + SagaConcurrencyError: If no row was updated (saga does not exist or was modified concurrently). """ - await self._session.execute( + result = await self._session.execute( sqlalchemy.update(SagaExecutionModel) .where(SagaExecutionModel.id == saga_id) .values( @@ -235,6 +238,10 @@ async def update_status( version=SagaExecutionModel.version + 1, ), ) + if result.rowcount == 0: + raise SagaConcurrencyError( + f"Saga {saga_id} does not exist or was modified concurrently", + ) async def log_step( self, @@ -378,7 +385,7 @@ async def _run() -> typing.AsyncGenerator[SagaStorageRun, None]: run = _SqlAlchemySagaStorageRun(session) try: yield run - except Exception: + except BaseException: await run.rollback() raise diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py index 92d34f1..a846b34 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_memory.py @@ -179,7 +179,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_memory_full_transaction( +def test_benchmark_saga_memory_run_full_transaction( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, @@ -206,7 +206,7 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_single_step( +def test_benchmark_saga_memory_run_single_step( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, diff --git a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py index d06739b..d189588 100644 --- a/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/dataclasses/test_benchmark_saga_sqlalchemy.py @@ -44,7 +44,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_full_transaction( +def test_benchmark_saga_sqlalchemy_run_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, @@ -75,7 +75,7 @@ async def run_transaction() -> None: @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_single_step( +def test_benchmark_saga_sqlalchemy_run_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, diff --git a/tests/benchmarks/default/test_benchmark_saga_memory.py b/tests/benchmarks/default/test_benchmark_saga_memory.py index 1f7856b..9c2aa96 100644 --- a/tests/benchmarks/default/test_benchmark_saga_memory.py +++ b/tests/benchmarks/default/test_benchmark_saga_memory.py @@ -162,10 +162,7 @@ def memory_storage_legacy() -> MemorySagaStorageLegacy: @pytest.fixture -def saga_with_memory_storage( - saga_container: SagaContainer, - memory_storage: MemorySagaStorage, -) -> Saga[OrderContext]: +def saga_with_memory_storage() -> Saga[OrderContext]: """ Create an OrderSaga preconfigured with inventory reservation, payment processing, and shipping steps. @@ -180,7 +177,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_memory_full_transaction( +def test_benchmark_saga_memory_run_full_transaction( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, @@ -207,7 +204,7 @@ async def run() -> None: @pytest.mark.benchmark -def test_benchmark_saga_memory_single_step( +def test_benchmark_saga_memory_run_single_step( benchmark, saga_with_memory_storage: Saga[OrderContext], saga_container: SagaContainer, diff --git a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py index 05b31d5..7a82ccf 100644 --- a/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py +++ b/tests/benchmarks/default/test_benchmark_saga_sqlalchemy.py @@ -20,6 +20,17 @@ ) +def _make_storage(engine, storage_cls): + """Build saga storage from engine and storage class (shared by legacy benchmarks).""" + session_factory = async_sessionmaker( + engine, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + return storage_cls(session_factory) + + @pytest.fixture def saga_container() -> SagaContainer: """ @@ -44,7 +55,7 @@ class OrderSaga(Saga[OrderContext]): @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_full_transaction( +def test_benchmark_saga_sqlalchemy_run_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, @@ -75,7 +86,7 @@ async def run_transaction() -> None: @pytest.mark.benchmark -def test_benchmark_saga_sqlalchemy_single_step( +def test_benchmark_saga_sqlalchemy_run_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, @@ -118,30 +129,24 @@ async def run_transaction() -> None: @pytest.mark.benchmark +@pytest.mark.parametrize( + "storage_cls", + [SqlAlchemySagaStorage, SqlAlchemySagaStorageLegacy], + ids=["storage", "legacy"], +) def test_benchmark_saga_sqlalchemy_legacy_full_transaction( benchmark, saga_sqlalchemy: Saga[OrderContext], saga_container: SagaContainer, saga_benchmark_loop_and_engine, + storage_cls, ): """Benchmark full saga transaction with SQLAlchemy storage, legacy path (MySQL).""" loop, engine = saga_benchmark_loop_and_engine - - session_factory = async_sessionmaker( - engine, - expire_on_commit=False, - autocommit=False, - autoflush=False, - ) - storage = SqlAlchemySagaStorageLegacy(session_factory) + storage = _make_storage(engine, storage_cls) context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: - """ - Execute the saga transaction and iterate through all produced steps without performing any operations. - - Opens the saga's transaction context with the configured container and storage, then consumes every yielded step (no-op per step). Intended for benchmarking the transaction iteration path. - """ async with saga_sqlalchemy.transaction( context=context, container=saga_container, @@ -154,10 +159,16 @@ async def run_transaction() -> None: @pytest.mark.benchmark +@pytest.mark.parametrize( + "storage_cls", + [SqlAlchemySagaStorage, SqlAlchemySagaStorageLegacy], + ids=["storage", "legacy"], +) def test_benchmark_saga_sqlalchemy_legacy_single_step( benchmark, saga_container: SagaContainer, saga_benchmark_loop_and_engine, + storage_cls, ): """Benchmark saga with single step, legacy path (SQLAlchemy storage).""" loop, engine = saga_benchmark_loop_and_engine @@ -166,22 +177,10 @@ class SingleStepSaga(Saga[OrderContext]): steps = [ReserveInventoryStep] saga = SingleStepSaga() - - session_factory = async_sessionmaker( - engine, - expire_on_commit=False, - autocommit=False, - autoflush=False, - ) - storage = SqlAlchemySagaStorageLegacy(session_factory) + storage = _make_storage(engine, storage_cls) context = OrderContext(order_id="ord_1", user_id="user_1", amount=100.0) async def run_transaction() -> None: - """ - Run the saga transaction to completion by iterating over its yielded steps using the configured context, container, and storage. - - This function is used by benchmarks to execute a full saga flow without performing additional work per step. - """ async with saga.transaction( context=context, container=saga_container, diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7059043..f4b6e6d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,8 +1,127 @@ +import typing + import pytest import redis from aiobreaker.storage.memory import CircuitMemoryStorage from aiobreaker.storage.redis import CircuitRedisStorage from aiobreaker import CircuitBreakerState +from unittest import mock + +import cqrs +from cqrs import events +from cqrs.requests.map import SagaMap +from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage + +from tests.integration.test_saga_mediator_memory import ( + FailingOrderSaga, + FailingStep, + InventoryReservedEvent, + InventoryReservedEventHandler, + OrderContext, + OrderShippedEvent, + OrderShippedEventHandler, + OrderSaga, + PaymentProcessedEvent, + PaymentProcessedEventHandler, + ProcessPaymentResponse, + ProcessPaymentStep, + ReserveInventoryResponse, + ReserveInventoryStep, + ShipOrderResponse, + ShipOrderStep, +) + + +class _TestContainer: + """Test container that resolves step handlers, sagas, and event handlers (shared by SQLAlchemy mediator tests).""" + + def __init__(self, storage: SqlAlchemySagaStorage) -> None: + self._storage = storage + self._external_container = None + self._step_handlers = { + ReserveInventoryStep: ReserveInventoryStep(), + ProcessPaymentStep: ProcessPaymentStep(), + ShipOrderStep: ShipOrderStep(), + FailingStep: FailingStep(), + } + self._event_handlers = { + InventoryReservedEventHandler: InventoryReservedEventHandler(), + PaymentProcessedEventHandler: PaymentProcessedEventHandler(), + OrderShippedEventHandler: OrderShippedEventHandler(), + } + self._sagas = { + OrderSaga: OrderSaga(), # type: ignore[arg-type] + FailingOrderSaga: FailingOrderSaga(), # type: ignore[arg-type] + } + + @property + def external_container(self) -> typing.Any: + return self._external_container + + def attach_external_container(self, container: typing.Any) -> None: + self._external_container = container + + async def resolve(self, type_) -> typing.Any: + if type_ in self._step_handlers: + return self._step_handlers[type_] + if type_ in self._event_handlers: + return self._event_handlers[type_] + if type_ in self._sagas: + return self._sagas[type_] + if type_ == SqlAlchemySagaStorage: + return self._storage + raise ValueError(f"Unknown type: {type_}") + + +@pytest.fixture +def container(storage: SqlAlchemySagaStorage) -> _TestContainer: + """Create test container (shared by SQLAlchemy mediator tests; requires storage fixture from test module).""" + container = _TestContainer(storage) + for step_handler in container._step_handlers.values(): + if hasattr(step_handler, "_events"): + step_handler._events.clear() + for event_handler in container._event_handlers.values(): + if hasattr(event_handler, "handled_events"): + event_handler.handled_events.clear() + return container + + +@pytest.fixture +def saga_mediator( + container: _TestContainer, + storage: SqlAlchemySagaStorage, +) -> cqrs.SagaMediator: + """Create SagaMediator with SqlAlchemySagaStorage (shared; storage comes from test module).""" + + def saga_mapper(mapper: SagaMap) -> None: + mapper.bind(OrderContext, OrderSaga) + + def events_mapper(mapper: events.EventMap) -> None: + mapper.bind(InventoryReservedEvent, InventoryReservedEventHandler) + mapper.bind(PaymentProcessedEvent, PaymentProcessedEventHandler) + mapper.bind(OrderShippedEvent, OrderShippedEventHandler) + + event_map = events.EventMap() + events_mapper(event_map) + message_broker = mock.AsyncMock() + message_broker.produce = mock.AsyncMock() + event_emitter = events.EventEmitter( + event_map=event_map, + container=container, # type: ignore + message_broker=message_broker, + ) + saga_map = SagaMap() + saga_mapper(saga_map) + mediator = cqrs.SagaMediator( + saga_map=saga_map, + container=container, # type: ignore + event_emitter=event_emitter, + event_map=event_map, + max_concurrent_event_handlers=2, + concurrent_event_handle_enable=True, + storage=storage, + ) + return mediator @pytest.fixture diff --git a/tests/integration/test_saga_mediator_sqlalchemy_mysql.py b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py index dd42fc2..fed3b00 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_mysql.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py @@ -1,28 +1,19 @@ """Integration tests for SagaMediator with SqlAlchemySagaStorage (MySQL).""" -import typing import uuid -from unittest import mock import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker import cqrs -from cqrs import events -from cqrs.requests.map import SagaMap from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from tests.integration.test_saga_mediator_memory import ( FailingOrderSaga, - FailingStep, - InventoryReservedEvent, InventoryReservedEventHandler, OrderContext, - OrderShippedEvent, OrderShippedEventHandler, - OrderSaga, - PaymentProcessedEvent, PaymentProcessedEventHandler, ProcessPaymentResponse, ProcessPaymentStep, @@ -33,47 +24,6 @@ ) -class _TestContainer: - """Test container that resolves step handlers, sagas, and event handlers.""" - - def __init__(self, storage: SqlAlchemySagaStorage) -> None: - self._storage = storage - self._external_container = None - self._step_handlers = { - ReserveInventoryStep: ReserveInventoryStep(), - ProcessPaymentStep: ProcessPaymentStep(), - ShipOrderStep: ShipOrderStep(), - FailingStep: FailingStep(), - } - self._event_handlers = { - InventoryReservedEventHandler: InventoryReservedEventHandler(), - PaymentProcessedEventHandler: PaymentProcessedEventHandler(), - OrderShippedEventHandler: OrderShippedEventHandler(), - } - self._sagas = { - OrderSaga: OrderSaga(), # type: ignore[arg-type] - FailingOrderSaga: FailingOrderSaga(), # type: ignore[arg-type] - } - - @property - def external_container(self) -> typing.Any: - return self._external_container - - def attach_external_container(self, container: typing.Any) -> None: - self._external_container = container - - async def resolve(self, type_) -> typing.Any: - if type_ in self._step_handlers: - return self._step_handlers[type_] - if type_ in self._event_handlers: - return self._event_handlers[type_] - if type_ in self._sagas: - return self._sagas[type_] - if type_ == SqlAlchemySagaStorage: - return self._storage - raise ValueError(f"Unknown type: {type_}") - - @pytest.fixture def storage( saga_session_factory_mysql: async_sessionmaker[AsyncSession], @@ -82,57 +32,6 @@ def storage( return SqlAlchemySagaStorage(saga_session_factory_mysql) -@pytest.fixture -def container(storage: SqlAlchemySagaStorage) -> _TestContainer: - """Create test container.""" - container = _TestContainer(storage) - for step_handler in container._step_handlers.values(): - if hasattr(step_handler, "_events"): - step_handler._events.clear() - for event_handler in container._event_handlers.values(): - if hasattr(event_handler, "handled_events"): - event_handler.handled_events.clear() - return container - - -@pytest.fixture -def saga_mediator( - container: _TestContainer, - storage: SqlAlchemySagaStorage, -) -> cqrs.SagaMediator: - """Create SagaMediator with SqlAlchemySagaStorage (MySQL).""" - - def saga_mapper(mapper: SagaMap) -> None: - mapper.bind(OrderContext, OrderSaga) - - def events_mapper(mapper: events.EventMap) -> None: - mapper.bind(InventoryReservedEvent, InventoryReservedEventHandler) - mapper.bind(PaymentProcessedEvent, PaymentProcessedEventHandler) - mapper.bind(OrderShippedEvent, OrderShippedEventHandler) - - event_map = events.EventMap() - events_mapper(event_map) - message_broker = mock.AsyncMock() - message_broker.produce = mock.AsyncMock() - event_emitter = events.EventEmitter( - event_map=event_map, - container=container, # type: ignore - message_broker=message_broker, - ) - saga_map = SagaMap() - saga_mapper(saga_map) - mediator = cqrs.SagaMediator( - saga_map=saga_map, - container=container, # type: ignore - event_emitter=event_emitter, - event_map=event_map, - max_concurrent_event_handlers=2, - concurrent_event_handle_enable=True, - storage=storage, - ) - return mediator - - class TestSagaMediatorSqlAlchemyStorageMysql: """Integration tests for SagaMediator with SqlAlchemySagaStorage (MySQL).""" diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py index 57605c9..4c12082 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -1,28 +1,19 @@ """Integration tests for SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" -import typing import uuid -from unittest import mock import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker import cqrs -from cqrs import events -from cqrs.requests.map import SagaMap from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from tests.integration.test_saga_mediator_memory import ( FailingOrderSaga, - FailingStep, - InventoryReservedEvent, InventoryReservedEventHandler, OrderContext, - OrderShippedEvent, OrderShippedEventHandler, - OrderSaga, - PaymentProcessedEvent, PaymentProcessedEventHandler, ProcessPaymentResponse, ProcessPaymentStep, @@ -33,47 +24,6 @@ ) -class _TestContainer: - """Test container that resolves step handlers, sagas, and event handlers.""" - - def __init__(self, storage: SqlAlchemySagaStorage) -> None: - self._storage = storage - self._external_container = None - self._step_handlers = { - ReserveInventoryStep: ReserveInventoryStep(), - ProcessPaymentStep: ProcessPaymentStep(), - ShipOrderStep: ShipOrderStep(), - FailingStep: FailingStep(), - } - self._event_handlers = { - InventoryReservedEventHandler: InventoryReservedEventHandler(), - PaymentProcessedEventHandler: PaymentProcessedEventHandler(), - OrderShippedEventHandler: OrderShippedEventHandler(), - } - self._sagas = { - OrderSaga: OrderSaga(), # type: ignore[arg-type] - FailingOrderSaga: FailingOrderSaga(), # type: ignore[arg-type] - } - - @property - def external_container(self) -> typing.Any: - return self._external_container - - def attach_external_container(self, container: typing.Any) -> None: - self._external_container = container - - async def resolve(self, type_) -> typing.Any: - if type_ in self._step_handlers: - return self._step_handlers[type_] - if type_ in self._event_handlers: - return self._event_handlers[type_] - if type_ in self._sagas: - return self._sagas[type_] - if type_ == SqlAlchemySagaStorage: - return self._storage - raise ValueError(f"Unknown type: {type_}") - - @pytest.fixture def storage( saga_session_factory_postgres: async_sessionmaker[AsyncSession], @@ -82,57 +32,6 @@ def storage( return SqlAlchemySagaStorage(saga_session_factory_postgres) -@pytest.fixture -def container(storage: SqlAlchemySagaStorage) -> _TestContainer: - """Create test container.""" - container = _TestContainer(storage) - for step_handler in container._step_handlers.values(): - if hasattr(step_handler, "_events"): - step_handler._events.clear() - for event_handler in container._event_handlers.values(): - if hasattr(event_handler, "handled_events"): - event_handler.handled_events.clear() - return container - - -@pytest.fixture -def saga_mediator( - container: _TestContainer, - storage: SqlAlchemySagaStorage, -) -> cqrs.SagaMediator: - """Create SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" - - def saga_mapper(mapper: SagaMap) -> None: - mapper.bind(OrderContext, OrderSaga) - - def events_mapper(mapper: events.EventMap) -> None: - mapper.bind(InventoryReservedEvent, InventoryReservedEventHandler) - mapper.bind(PaymentProcessedEvent, PaymentProcessedEventHandler) - mapper.bind(OrderShippedEvent, OrderShippedEventHandler) - - event_map = events.EventMap() - events_mapper(event_map) - message_broker = mock.AsyncMock() - message_broker.produce = mock.AsyncMock() - event_emitter = events.EventEmitter( - event_map=event_map, - container=container, # type: ignore - message_broker=message_broker, - ) - saga_map = SagaMap() - saga_mapper(saga_map) - mediator = cqrs.SagaMediator( - saga_map=saga_map, - container=container, # type: ignore - event_emitter=event_emitter, - event_map=event_map, - max_concurrent_event_handlers=2, - concurrent_event_handle_enable=True, - storage=storage, - ) - return mediator - - class TestSagaMediatorSqlAlchemyStoragePostgres: """Integration tests for SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" diff --git a/tests/integration/test_saga_storage_sqlalchemy_mysql.py b/tests/integration/test_saga_storage_sqlalchemy_mysql.py index 7f691e7..a39cb15 100644 --- a/tests/integration/test_saga_storage_sqlalchemy_mysql.py +++ b/tests/integration/test_saga_storage_sqlalchemy_mysql.py @@ -23,7 +23,7 @@ def storage( saga_session_factory_mysql: async_sessionmaker[AsyncSession], ) -> SqlAlchemySagaStorage: - """SqlAlchemySagaStorage для MySQL (фикстура init_saga_orm_mysql поднимает схему).""" + """SqlAlchemySagaStorage for MySQL (the init_saga_orm_mysql fixture sets up the schema).""" return SqlAlchemySagaStorage(saga_session_factory_mysql) diff --git a/tests/integration/test_saga_storage_sqlalchemy_postgres.py b/tests/integration/test_saga_storage_sqlalchemy_postgres.py index 9ab308c..88d99c8 100644 --- a/tests/integration/test_saga_storage_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_storage_sqlalchemy_postgres.py @@ -5,9 +5,10 @@ import asyncio import uuid from collections.abc import AsyncGenerator +from datetime import datetime, timedelta, timezone import pytest -from sqlalchemy import delete +from sqlalchemy import delete, update from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from cqrs.dispatcher.exceptions import SagaConcurrencyError @@ -23,7 +24,7 @@ def storage( saga_session_factory_postgres: async_sessionmaker[AsyncSession], ) -> SqlAlchemySagaStorage: - """SqlAlchemySagaStorage для PostgreSQL (фикстура init_saga_orm_postgres поднимает схему).""" + """SqlAlchemySagaStorage for PostgreSQL (the init_saga_orm_postgres fixture sets up the schema).""" return SqlAlchemySagaStorage(saga_session_factory_postgres) @@ -248,8 +249,15 @@ async def test_get_sagas_for_recovery_ordered_by_updated_at( for sid in (id1, id2, id3): await storage.create_saga(saga_id=sid, name="saga", context=test_context) await storage.update_status(sid, SagaStatus.RUNNING) - await asyncio.sleep(1.0) - await storage.update_context(id2, {**test_context, "touched": True}) + # Set id2's updated_at to a later time so ordering is deterministic (no sleep). + later = datetime.now(timezone.utc) + timedelta(seconds=10) + async with storage.session_factory() as session: + await session.execute( + update(SagaExecutionModel) + .where(SagaExecutionModel.id == id2) + .values(updated_at=later), + ) + await session.commit() ids = await storage.get_sagas_for_recovery(limit=10) assert len(ids) == 3 assert ids[-1] == id2 From cf26135ec9532c4aae625c9da1f64b7305b1e245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:13:05 +0300 Subject: [PATCH 10/17] fix after review --- tests/integration/conftest.py | 3 --- tests/integration/test_saga_mediator_sqlalchemy_postgres.py | 5 ++++- tests/integration/test_saga_storage_sqlalchemy_postgres.py | 5 +---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f4b6e6d..562a0be 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,11 +23,8 @@ OrderSaga, PaymentProcessedEvent, PaymentProcessedEventHandler, - ProcessPaymentResponse, ProcessPaymentStep, - ReserveInventoryResponse, ReserveInventoryStep, - ShipOrderResponse, ShipOrderStep, ) diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py index 4c12082..9912d21 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -1,19 +1,22 @@ """Integration tests for SagaMediator with SqlAlchemySagaStorage (PostgreSQL).""" import uuid +from unittest import mock import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker import cqrs +from cqrs import events, SagaMap from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage +from integration.conftest import _TestContainer from tests.integration.test_saga_mediator_memory import ( FailingOrderSaga, InventoryReservedEventHandler, OrderContext, - OrderShippedEventHandler, + OrderSaga, OrderShippedEventHandler, PaymentProcessedEventHandler, ProcessPaymentResponse, ProcessPaymentStep, diff --git a/tests/integration/test_saga_storage_sqlalchemy_postgres.py b/tests/integration/test_saga_storage_sqlalchemy_postgres.py index 88d99c8..24ac125 100644 --- a/tests/integration/test_saga_storage_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_storage_sqlalchemy_postgres.py @@ -2,7 +2,6 @@ Uses DATABASE_DSN_POSTGRESQL from fixtures (pytest-config.ini / env). """ -import asyncio import uuid from collections.abc import AsyncGenerator from datetime import datetime, timedelta, timezone @@ -253,9 +252,7 @@ async def test_get_sagas_for_recovery_ordered_by_updated_at( later = datetime.now(timezone.utc) + timedelta(seconds=10) async with storage.session_factory() as session: await session.execute( - update(SagaExecutionModel) - .where(SagaExecutionModel.id == id2) - .values(updated_at=later), + update(SagaExecutionModel).where(SagaExecutionModel.id == id2).values(updated_at=later), ) await session.commit() ids = await storage.get_sagas_for_recovery(limit=10) From 1e5a0bb6b63c86a84b753c17ced5c3d10f155219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:13:39 +0300 Subject: [PATCH 11/17] fix after review --- tests/integration/test_saga_mediator_sqlalchemy_mysql.py | 6 ++++-- tests/integration/test_saga_mediator_sqlalchemy_postgres.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_saga_mediator_sqlalchemy_mysql.py b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py index fed3b00..0b82746 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_mysql.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py @@ -1,19 +1,21 @@ """Integration tests for SagaMediator with SqlAlchemySagaStorage (MySQL).""" import uuid +from unittest import mock import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker import cqrs +from cqrs import events, SagaMap from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from tests.integration.test_saga_mediator_memory import ( - FailingOrderSaga, + _TestContainer, FailingOrderSaga, InventoryReservedEventHandler, OrderContext, - OrderShippedEventHandler, + OrderSaga, OrderShippedEventHandler, PaymentProcessedEventHandler, ProcessPaymentResponse, ProcessPaymentStep, diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py index 9912d21..a213134 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -16,7 +16,8 @@ FailingOrderSaga, InventoryReservedEventHandler, OrderContext, - OrderSaga, OrderShippedEventHandler, + OrderSaga, + OrderShippedEventHandler, PaymentProcessedEventHandler, ProcessPaymentResponse, ProcessPaymentStep, From b3453320871110aa5e7b09ea20fdd5cba5c80b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:15:29 +0300 Subject: [PATCH 12/17] fix after review --- tests/integration/test_saga_mediator_sqlalchemy_mysql.py | 6 ++++-- tests/integration/test_saga_mediator_sqlalchemy_postgres.py | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_saga_mediator_sqlalchemy_mysql.py b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py index 0b82746..833b2d6 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_mysql.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_mysql.py @@ -12,10 +12,12 @@ from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from tests.integration.test_saga_mediator_memory import ( - _TestContainer, FailingOrderSaga, + _TestContainer, + FailingOrderSaga, InventoryReservedEventHandler, OrderContext, - OrderSaga, OrderShippedEventHandler, + OrderSaga, + OrderShippedEventHandler, PaymentProcessedEventHandler, ProcessPaymentResponse, ProcessPaymentStep, diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py index a213134..051fd7c 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -10,10 +10,9 @@ from cqrs import events, SagaMap from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage -from integration.conftest import _TestContainer from tests.integration.test_saga_mediator_memory import ( - FailingOrderSaga, + _TestContainer, FailingOrderSaga, InventoryReservedEventHandler, OrderContext, OrderSaga, From df2dabbcb7642816b4b44bf135b1061aa8e5d8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:15:47 +0300 Subject: [PATCH 13/17] fix after review --- tests/integration/test_saga_mediator_sqlalchemy_postgres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py index 051fd7c..611699f 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -12,7 +12,8 @@ from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage from tests.integration.test_saga_mediator_memory import ( - _TestContainer, FailingOrderSaga, + _TestContainer, + FailingOrderSaga, InventoryReservedEventHandler, OrderContext, OrderSaga, From 64557cb29217de0e32c47d64f4727a2952a6a7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:25:26 +0300 Subject: [PATCH 14/17] fix after review --- examples/saga_sqlalchemy_storage.py | 4 ++-- src/cqrs/saga/saga.py | 8 ++++++-- src/cqrs/saga/storage/sqlalchemy.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/saga_sqlalchemy_storage.py b/examples/saga_sqlalchemy_storage.py index 3b14b46..0260fa9 100644 --- a/examples/saga_sqlalchemy_storage.py +++ b/examples/saga_sqlalchemy_storage.py @@ -40,8 +40,8 @@ logger = logging.getLogger(__name__) # Database Configuration -# Using SQLite for this example, but can be swapped for PostgreSQL/MySQL -DB_URL = os.getenv("DATABASE_URL", "mysql+asyncmy://cqrs:cqrs@localhost:3307/test_cqrs") +# Using SQLite for this example, but can be swapped for PostgreSQL/MySQL via DATABASE_URL +DB_URL = os.getenv("DATABASE_URL", "sqlite+aiosqlite:///./test.db") # ============================================================================ diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index db5a3b1..2d7658a 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -168,8 +168,12 @@ async def __aiter__( run_cm = None if run_cm is not None: - async with run_cm as run: - async for step_result in self._execute(run): + try: + async with run_cm as run: + async for step_result in self._execute(run): + yield step_result + except NotImplementedError: + async for step_result in self._execute(None): yield step_result else: async for step_result in self._execute(None): diff --git a/src/cqrs/saga/storage/sqlalchemy.py b/src/cqrs/saga/storage/sqlalchemy.py index a1a4ae7..99e2d58 100644 --- a/src/cqrs/saga/storage/sqlalchemy.py +++ b/src/cqrs/saga/storage/sqlalchemy.py @@ -238,7 +238,7 @@ async def update_status( version=SagaExecutionModel.version + 1, ), ) - if result.rowcount == 0: + if result.rowcount == 0: # pyright: ignore[reportAttributeAccessIssue] raise SagaConcurrencyError( f"Saga {saga_id} does not exist or was modified concurrently", ) From d77c49ca54a032c7d5411b30cb52ae58798ddb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Fri, 20 Feb 2026 14:39:28 +0300 Subject: [PATCH 15/17] fix after review --- src/cqrs/saga/saga.py | 8 ++------ .../integration/test_saga_mediator_sqlalchemy_postgres.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/cqrs/saga/saga.py b/src/cqrs/saga/saga.py index 2d7658a..db5a3b1 100644 --- a/src/cqrs/saga/saga.py +++ b/src/cqrs/saga/saga.py @@ -168,12 +168,8 @@ async def __aiter__( run_cm = None if run_cm is not None: - try: - async with run_cm as run: - async for step_result in self._execute(run): - yield step_result - except NotImplementedError: - async for step_result in self._execute(None): + async with run_cm as run: + async for step_result in self._execute(run): yield step_result else: async for step_result in self._execute(None): diff --git a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py index 611699f..bb93dfd 100644 --- a/tests/integration/test_saga_mediator_sqlalchemy_postgres.py +++ b/tests/integration/test_saga_mediator_sqlalchemy_postgres.py @@ -11,8 +11,8 @@ from cqrs.saga.storage.enums import SagaStatus from cqrs.saga.storage.sqlalchemy import SqlAlchemySagaStorage +from tests.integration.conftest import _TestContainer from tests.integration.test_saga_mediator_memory import ( - _TestContainer, FailingOrderSaga, InventoryReservedEventHandler, OrderContext, From f3deef2c8d48964f60d9bcb3a987c946ef8de670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Sat, 21 Feb 2026 13:32:30 +0300 Subject: [PATCH 16/17] refactor readme --- README.md | 291 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 224 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 5863766..6bf63d9 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ -
Python CQRS -

Python CQRS

-

Event-Driven Architecture Framework for Distributed Systems

+

Event-Driven Architecture Framework for Distributed Systems

+

+ Python 3.10+ · Full documentation: mkdocs.python-cqrs.dev +

Python Versions @@ -41,6 +42,28 @@ > > Starting with version 5.0.0, Pydantic support will become optional. The default implementations of `Request`, `Response`, `DomainEvent`, and `NotificationEvent` will be migrated to dataclasses-based implementations. +## Table of Contents + +- [Overview](#overview) +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Request Handlers](#request-handlers) +- [Request and Response Types](#request-and-response-types) +- [Mapping](#mapping) +- [Bootstrap](#bootstrap) +- [Saga Pattern](#saga-pattern) +- [Event Handlers](#event-handlers) +- [Producing Notification Events](#producing-notification-events) +- [Kafka broker](#kafka-broker) +- [Transactional Outbox](#transactional-outbox) +- [Producing Events from Outbox to Kafka](#producing-events-from-outbox-to-kafka) +- [DI container](#di-container) +- [Integration with presentation layers](#integration-with-presentation-layers) +- [Protobuf messaging](#protobuf-messaging) +- [Contributing](#contributing) +- [Changelog](#changelog) +- [License](#license) + ## Overview An event-driven framework for building distributed systems in Python. It centers on CQRS (Command Query Responsibility Segregation) and extends into messaging, sagas, and reliable event delivery — so you can separate read and write flows, react to events from the bus, run distributed transactions with compensation, and publish events via Transaction Outbox. The result is clearer structure, better scalability, and easier evolution of the application. @@ -67,6 +90,49 @@ project ([documentation](https://akhundmurad.github.io/diator/)) with several en - **Documentation:** Built-in Mermaid diagram generation (Sequence and Class diagrams). - **Protobuf:** Interface-level support for converting Notification events to Protobuf and back. +## Installation + +**Python 3.10+** is required. + +```bash +pip install python-cqrs +``` + +Optional dependencies (see [pyproject.toml](https://github.com/vadikko2/python-cqrs/blob/master/pyproject.toml) for full list): + +```bash +pip install python-cqrs[kafka] # Kafka broker (aiokafka) +pip install python-cqrs[examples] # FastAPI, FastStream, uvicorn, etc. +pip install python-cqrs[aiobreaker] # Circuit breaker for saga fallbacks +``` + +## Quick Start + +Define a command, a handler, bind them, and run via the mediator: + +```python +import di +import cqrs +from cqrs.requests import bootstrap + +class CreateOrderCommand(cqrs.Request): + order_id: str + amount: float + +class CreateOrderHandler(cqrs.RequestHandler[CreateOrderCommand, None]): + async def handle(self, request: CreateOrderCommand) -> None: + print(f"Order {request.order_id}, amount {request.amount}") + +def commands_mapper(mapper: cqrs.RequestMap) -> None: + mapper.bind(CreateOrderCommand, CreateOrderHandler) + +container = di.Container() +mediator = bootstrap.bootstrap(di_container=container, commands_mapper=commands_mapper) +await mediator.send(CreateOrderCommand(order_id="ord-1", amount=99.99)) +``` + +For full setup with DI, events, and outbox, see the [documentation](https://mkdocs.python-cqrs.dev/) and the [examples](https://github.com/vadikko2/python-cqrs/tree/master/examples) directory. + ## Request Handlers Request handlers can be divided into two main types: @@ -98,7 +164,7 @@ class JoinMeetingCommandHandler(RequestHandler[JoinMeetingCommand, None]): ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/request_handler.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/request_handler.py) ### Query handler @@ -128,7 +194,7 @@ class ReadMeetingQueryHandler(RequestHandler[ReadMeetingQuery, ReadMeetingQueryR ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/request_handler.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/request_handler.py) ### Streaming Request Handler @@ -164,7 +230,7 @@ class ProcessFilesCommandHandler(StreamingRequestHandler[ProcessFilesCommand, Fi ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/streaming_handler_parallel_events.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/streaming_handler_parallel_events.py) ### Chain of Responsibility Request Handler @@ -227,7 +293,7 @@ def payment_mapper(mapper: cqrs.RequestMap) -> None: ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/cor_request_handler.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/cor_request_handler.py) #### Mermaid Diagram Generation @@ -247,7 +313,7 @@ sequence_diagram = generator.sequence() class_diagram = generator.class_diagram() ``` -Complete example: [CoR Mermaid Diagrams](https://github.com/vadikko2/cqrs/blob/master/examples/cor_mermaid.py) +Complete example: [CoR Mermaid Diagrams](https://github.com/vadikko2/python-cqrs/blob/master/examples/cor_mermaid.py) ## Request and Response Types @@ -302,11 +368,13 @@ class CustomResponse(cqrs.IResponse): return cls(result=kwargs["result"], status=kwargs["status"]) ``` -A complete example can be found in [request_response_types.py](https://github.com/vadikko2/cqrs/blob/master/examples/request_response_types.py) +A complete example can be found in [request_response_types.py](https://github.com/vadikko2/python-cqrs/blob/master/examples/request_response_types.py) ## Mapping -To bind commands, queries and events with specific handlers, you can use the registries `EventMap` and `RequestMap`. +To bind commands, queries and events with specific handlers, you can use the registries `EventMap`, `RequestMap`, and `SagaMap`. + +**Commands, queries and events:** ```python from cqrs import requests, events @@ -327,6 +395,35 @@ def init_events(mapper: events.EventMap) -> None: mapper.bind(events.NotificationEvent[event_models.ECSTMeetingRoomClosed], event_handlers.UpdateMeetingRoomReadModelHandler) ``` +**Chain of Responsibility** — bind a list of handlers (the first one that can handle the request processes it, otherwise the request is passed to the next): + +```python +def payment_mapper(mapper: cqrs.RequestMap) -> None: + mapper.bind( + ProcessPaymentCommand, + [ + CreditCardPaymentHandler, + PayPalPaymentHandler, + DefaultPaymentHandler, # Fallback + ], + ) +``` + +**Streaming handler** — bind a command to a `StreamingRequestHandler` (results are yielded as they become available): + +```python +def commands_mapper(mapper: cqrs.RequestMap) -> None: + mapper.bind(ProcessOrdersCommand, ProcessOrdersCommandHandler) # StreamingRequestHandler +``` + +**Saga (including with fallback)** — bind the saga context type to the saga class in `SagaMap`: + +```python +def saga_mapper(mapper: cqrs.SagaMap) -> None: + mapper.bind(OrderContext, OrderSaga) + mapper.bind(OrderContext, OrderSagaWithFallback) +``` + ## Bootstrap The `python-cqrs` package implements a set of bootstrap utilities designed to simplify the initial configuration of an @@ -359,6 +456,16 @@ def event_mediator_factory(): events_mapper=mapping.init_events, on_startup=[orm.init_store_event_mapper], ) + + +@functools.lru_cache +def saga_mediator_factory(): + return saga_bootstrap.bootstrap( + di_container=dependencies.setup_di(), + sagas_mapper=mapping.init_sagas, + domain_events_mapper=mapping.init_events, + saga_storage=MemorySagaStorage(), + ) ``` ## Saga Pattern @@ -512,7 +619,7 @@ sequence_diagram = generator.sequence() class_diagram = generator.class_diagram() ``` -Complete example: [Saga Mermaid Diagrams](https://github.com/vadikko2/cqrs/blob/master/examples/saga_mermaid.py) +Complete example: [Saga Mermaid Diagrams](https://github.com/vadikko2/python-cqrs/blob/master/examples/saga_mermaid.py) ## Event Handlers @@ -543,7 +650,7 @@ class UserJoinedEventHandler(cqrs.EventHandler[UserJoined]): ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/domain_event_handler.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/domain_event_handler.py) ### Parallel Event Processing @@ -613,7 +720,7 @@ class JoinMeetingCommandHandler(cqrs.RequestHandler[JoinMeetingCommand, None]): ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/event_producing.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/event_producing.py) After processing the command/request, if there are any Notification/ECST events, the EventEmitter is invoked to produce the events via the message broker. @@ -648,13 +755,6 @@ The package implements the [Transactional Outbox](https://microservices.io/patte pattern, which ensures that messages are produced to the broker according to the at-least-once semantics. ```python -def do_some_logic(meeting_room_id: int, session: sql_session.AsyncSession): - """ - Make changes to the database - """ - session.add(...) - - class JoinMeetingCommandHandler(cqrs.RequestHandler[JoinMeetingCommand, None]): def __init__(self, outbox: cqrs.OutboxedEventRepository): self.outbox = outbox @@ -665,35 +765,33 @@ class JoinMeetingCommandHandler(cqrs.RequestHandler[JoinMeetingCommand, None]): async def handle(self, request: JoinMeetingCommand) -> None: print(f"User {request.user_id} joined meeting {request.meeting_id}") - async with self.outbox as session: - do_some_logic(request.meeting_id, session) # business logic - self.outbox.add( - session, - cqrs.NotificationEvent[UserJoinedNotificationPayload]( - event_name="UserJoined", - topic="user_notification_events", - payload=UserJoinedNotificationPayload( - user_id=request.user_id, - meeting_id=request.meeting_id, - ), + # Outbox repository is bound to a session (e.g. via DI request scope). + # add() takes only the event; commit() persists the outbox and your changes. + self.outbox.add( + cqrs.NotificationEvent[UserJoinedNotificationPayload]( + event_name="UserJoined", + topic="user_notification_events", + payload=UserJoinedNotificationPayload( + user_id=request.user_id, + meeting_id=request.meeting_id, ), - ) - self.outbox.add( - session, - cqrs.NotificationEvent[UserJoinedECSTPayload]( - event_name="UserJoined", - topic="user_ecst_events", - payload=UserJoinedECSTPayload( - user_id=request.user_id, - meeting_id=request.meeting_id, - ), + ), + ) + self.outbox.add( + cqrs.NotificationEvent[UserJoinedECSTPayload]( + event_name="UserJoined", + topic="user_ecst_events", + payload=UserJoinedECSTPayload( + user_id=request.user_id, + meeting_id=request.meeting_id, ), - ) - await self.outbox.commit(session) + ), + ) + await self.outbox.commit() ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/save_events_into_outbox.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/save_events_into_outbox.py) > [!TIP] > You can specify the name of the Outbox table using the environment variable `OUTBOX_SQLA_TABLE`. @@ -701,8 +799,8 @@ the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/save_e > [!TIP] > If you use the protobuf events you should specify `OutboxedEventRepository` -> by [protobuf serialize](https://github.com/vadikko2/cqrs/blob/master/src/cqrs/serializers/protobuf.py). A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/save_proto_events_into_outbox.py) +> by [protobuf serialize](https://github.com/vadikko2/python-cqrs/blob/master/src/cqrs/serializers/protobuf.py). A complete example can be found in +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/save_proto_events_into_outbox.py) ## Producing Events from Outbox to Kafka @@ -728,23 +826,20 @@ broker = kafka.KafkaMessageBroker( producer=kafka_adapters.kafka_producer_factory(dsn="localhost:9092"), ) -producer = cqrs.EventProducer(broker, cqrs.SqlAlchemyOutboxedEventRepository(session_factory, zlib.ZlibCompressor())) - - -async def periodically_task(): - async for messages in producer.event_batch_generator(): - for message in messages: - await producer.send_message(message) - await producer.repository.commit() - await asyncio.sleep(10) +# SqlAlchemyOutboxedEventRepository expects (session, compressor), not session_factory. +async with session_factory() as session: + repository = cqrs.SqlAlchemyOutboxedEventRepository(session, zlib.ZlibCompressor()) + producer = cqrs.EventProducer(broker, repository) - -loop = asyncio.get_event_loop() -loop.run_until_complete(periodically_task()) + async for messages in producer.event_batch_generator(): + for message in messages: + await producer.send_message(message) + await producer.repository.commit() + await asyncio.sleep(10) ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/kafka_outboxed_event_producing.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/kafka_outboxed_event_producing.py) **Transaction log tailing.** If Outbox polling does not suit you, consider [Transaction Log Tailing](https://microservices.io/patterns/data/transaction-log-tailing.html). The package does not implement it; you can use [Debezium + Kafka Connect](https://debezium.io/documentation/reference/stable/architecture.html) to tail the Outbox and produce events to Kafka. @@ -782,7 +877,7 @@ def setup_di() -> di.Container: ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injection.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injection.py) ### dependency-injector library @@ -809,8 +904,8 @@ mediator = bootstrap.bootstrap( ``` Complete examples can be found in: -- [Simple example](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injector_integration_simple_example.py) -- [Practical example with FastAPI](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injector_integration_practical_example.py) +- [Simple example](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injector_integration_simple_example.py) +- [Practical example with FastAPI](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injector_integration_practical_example.py) ## Integration with presentation layers @@ -832,7 +927,7 @@ In this case you can use python-cqrs to route requests to the appropriate handle import fastapi import pydantic -from app import dependecies, commands +from app import dependencies, commands router = fastapi.APIRouter(prefix="/meetings") @@ -848,7 +943,7 @@ async def join_metting( ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/fastapi_integration.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/fastapi_integration.py) ### Kafka events consuming @@ -944,9 +1039,71 @@ async def process_files_stream( ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/fastapi_sse_streaming.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/fastapi_sse_streaming.py) ## Protobuf messaging The `python-cqrs` package supports integration with [protobuf](https://developers.google.com/protocol-buffers/). -There is interface-level support for converting Notification events to Protobuf and back. Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data – think XML, but smaller, faster, and simpler. You define how you want your data to be structured once, then you can use special generated source code to easily write and read your structured data to and from a variety of data streams and using a variety of languages. +Notification events can be serialized to Protobuf and back: implement the `proto()` method (returns a protobuf message) and the class method `from_proto()` (creates an event instance from proto) on your event class. + +Example (assuming generated `user_joined_pb2` from your `.proto` with fields `event_id`, `event_timestamp`, `event_name`, `payload`): + +```python +import uuid +from datetime import datetime + +import cqrs +from app.generated import user_joined_pb2 # generated from .proto + + +class UserJoinedPayload(cqrs.Response): + user_id: str + meeting_id: str + + +class UserJoinedNotificationEvent(cqrs.NotificationEvent[UserJoinedPayload]): + """Event with Protobuf serialization support.""" + + event_name: str = "UserJoined" + + def proto(self): + msg = user_joined_pb2.UserJoinedNotification() + msg.event_id = str(self.event_id) + msg.event_timestamp = self.event_timestamp.isoformat() + msg.event_name = self.event_name + msg.payload.user_id = self.payload.user_id + msg.payload.meeting_id = self.payload.meeting_id + return msg + + @classmethod + def from_proto(cls, proto_msg): + return cls( + event_id=uuid.UUID(proto_msg.event_id), + event_timestamp=datetime.fromisoformat(proto_msg.event_timestamp), + event_name=proto_msg.event_name, + topic="user_notification_events", + payload=UserJoinedPayload( + user_id=proto_msg.payload.user_id, + meeting_id=proto_msg.payload.meeting_id, + ), + ) +``` + +## Contributing + +Contributions are welcome. To develop locally: + +1. Clone the repository and create a virtual environment. +2. Install dev dependencies: `pip install -e ".[dev]"`. +3. Run tests: `pytest`. +4. Install pre-commit and run hooks: `pre-commit install && pre-commit run --all-files`. + +The project uses [ruff](https://docs.astral.sh/ruff/) for linting and [pyright](https://microsoft.github.io/pyright/) for type checking. + +## Changelog + +Release notes and migration guides are published on [GitHub Releases](https://github.com/vadikko2/python-cqrs/releases). + +## License + +This project is licensed under the MIT License — see the [LICENSE](LICENSE) file for details. From f272de72f590f9fb3b4d9dd59f073cc53649c127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=9A=D0=BE=D0=B7=D1=8B?= =?UTF-8?q?=D1=80=D0=B5=D0=B2=D1=81=D0=BA=D0=B8=D0=B9?= Date: Sat, 21 Feb 2026 13:57:24 +0300 Subject: [PATCH 17/17] refactor readme --- README.md | 388 +++++++++++++++++++++++------------------------------- 1 file changed, 166 insertions(+), 222 deletions(-) diff --git a/README.md b/README.md index 6e3ce4c..67d71d2 100644 --- a/README.md +++ b/README.md @@ -47,17 +47,18 @@ - [Overview](#overview) - [Installation](#installation) - [Quick Start](#quick-start) -- [Request Handlers](#request-handlers) - [Request and Response Types](#request-and-response-types) +- [Request Handlers](#request-handlers) - [Mapping](#mapping) +- [DI container](#di-container) - [Bootstrap](#bootstrap) - [Saga Pattern](#saga-pattern) -- [Event Handlers](#event-handlers) - [Producing Notification Events](#producing-notification-events) - [Kafka broker](#kafka-broker) - [Transactional Outbox](#transactional-outbox) - [Producing Events from Outbox to Kafka](#producing-events-from-outbox-to-kafka) -- [DI container](#di-container) +- [Transaction log tailing](#transaction-log-tailing) +- [Event Handlers](#event-handlers) - [Integration with presentation layers](#integration-with-presentation-layers) - [Protobuf messaging](#protobuf-messaging) - [Contributing](#contributing) @@ -133,6 +134,61 @@ await mediator.send(CreateOrderCommand(order_id="ord-1", amount=99.99)) For full setup with DI, events, and outbox, see the [documentation](https://mkdocs.python-cqrs.dev/) and the [examples](https://github.com/vadikko2/python-cqrs/tree/master/examples) directory. +## Request and Response Types + +The library supports both Pydantic-based (`PydanticRequest`/`PydanticResponse`, aliased as `Request`/`Response`) and Dataclass-based (`DCRequest`/`DCResponse`) implementations. You can also implement custom classes by implementing the `IRequest`/`IResponse` interfaces directly. + +```python +import dataclasses + +# Pydantic-based (default) +class CreateUserCommand(cqrs.Request): + username: str + email: str + +class UserResponse(cqrs.Response): + user_id: str + username: str + +# Dataclass-based +@dataclasses.dataclass +class CreateProductCommand(cqrs.DCRequest): + name: str + price: float + +@dataclasses.dataclass +class ProductResponse(cqrs.DCResponse): + product_id: str + name: str + +# Custom implementation +class CustomRequest(cqrs.IRequest): + def __init__(self, user_id: str, action: str): + self.user_id = user_id + self.action = action + + def to_dict(self) -> dict: + return {"user_id": self.user_id, "action": self.action} + + @classmethod + def from_dict(cls, **kwargs) -> "CustomRequest": + return cls(user_id=kwargs["user_id"], action=kwargs["action"]) + +class CustomResponse(cqrs.IResponse): + def __init__(self, result: str, status: int): + self.result = result + self.status = status + + def to_dict(self) -> dict: + return {"result": self.result, "status": self.status} + + @classmethod + def from_dict(cls, **kwargs) -> "CustomResponse": + return cls(result=kwargs["result"], status=kwargs["status"]) +``` + +A complete example can be found in [request_response_types.py](https://github.com/vadikko2/python-cqrs/blob/master/examples/request_response_types.py) + ## Request Handlers Request handlers can be divided into two main types: @@ -315,61 +371,6 @@ class_diagram = generator.class_diagram() Complete example: [CoR Mermaid Diagrams](https://github.com/vadikko2/python-cqrs/blob/master/examples/cor_mermaid.py) -## Request and Response Types - -The library supports both Pydantic-based (`PydanticRequest`/`PydanticResponse`, aliased as `Request`/`Response`) and Dataclass-based (`DCRequest`/`DCResponse`) implementations. You can also implement custom classes by implementing the `IRequest`/`IResponse` interfaces directly. - -```python -import dataclasses - -# Pydantic-based (default) -class CreateUserCommand(cqrs.Request): - username: str - email: str - -class UserResponse(cqrs.Response): - user_id: str - username: str - -# Dataclass-based -@dataclasses.dataclass -class CreateProductCommand(cqrs.DCRequest): - name: str - price: float - -@dataclasses.dataclass -class ProductResponse(cqrs.DCResponse): - product_id: str - name: str - -# Custom implementation -class CustomRequest(cqrs.IRequest): - def __init__(self, user_id: str, action: str): - self.user_id = user_id - self.action = action - - def to_dict(self) -> dict: - return {"user_id": self.user_id, "action": self.action} - - @classmethod - def from_dict(cls, **kwargs) -> "CustomRequest": - return cls(user_id=kwargs["user_id"], action=kwargs["action"]) - -class CustomResponse(cqrs.IResponse): - def __init__(self, result: str, status: int): - self.result = result - self.status = status - - def to_dict(self) -> dict: - return {"result": self.result, "status": self.status} - - @classmethod - def from_dict(cls, **kwargs) -> "CustomResponse": - return cls(result=kwargs["result"], status=kwargs["status"]) -``` - -A complete example can be found in [request_response_types.py](https://github.com/vadikko2/python-cqrs/blob/master/examples/request_response_types.py) - ## Mapping To bind commands, queries and events with specific handlers, you can use the registries `EventMap`, `RequestMap`, and `SagaMap`. @@ -424,6 +425,70 @@ def saga_mapper(mapper: cqrs.SagaMap) -> None: mapper.bind(OrderContext, OrderSagaWithFallback) ``` +## DI container + +Use the following example to set up dependency injection in your command, query and event handlers. This will make +dependency management simpler. + +The package supports two DI container libraries: + +### di library + +```python +import di +... + +def setup_di() -> di.Container: + """ + Binds implementations to dependencies + """ + container = di.Container() + container.bind( + di.bind_by_type( + dependent.Dependent(cqrs.SqlAlchemyOutboxedEventRepository, scope="request"), + cqrs.OutboxedEventRepository + ) + ) + container.bind( + di.bind_by_type( + dependent.Dependent(MeetingAPIImplementaion, scope="request"), + MeetingAPIProtocol + ) + ) + return container +``` + +A complete example can be found in +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injection.py) + +### dependency-injector library + +The package also supports [dependency-injector](https://github.com/ets-labs/python-dependency-injector) library. +You can use `DependencyInjectorCQRSContainer` adapter to integrate dependency-injector containers with python-cqrs. + +```python +from dependency_injector import containers, providers +from cqrs.container.dependency_injector import DependencyInjectorCQRSContainer + +class ApplicationContainer(containers.DeclarativeContainer): + # Define your providers + service = providers.Factory(ServiceImplementation) + +# Create CQRS container adapter +cqrs_container = DependencyInjectorCQRSContainer(ApplicationContainer()) + +# Use with bootstrap +mediator = bootstrap.bootstrap( + di_container=cqrs_container, + commands_mapper=commands_mapper, + ... +) +``` + +Complete examples can be found in: +- [Simple example](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injector_integration_simple_example.py) +- [Practical example with FastAPI](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injector_integration_practical_example.py) + ## Bootstrap The `python-cqrs` package implements a set of bootstrap utilities designed to simplify the initial configuration of an @@ -621,67 +686,6 @@ class_diagram = generator.class_diagram() Complete example: [Saga Mermaid Diagrams](https://github.com/vadikko2/python-cqrs/blob/master/examples/saga_mermaid.py) -## Event Handlers - -Event handlers are designed to process `Notification` and `ECST` events that are consumed from the broker. -To configure event handling, you need to implement a broker consumer on the side of your application. -Below is an example of `Kafka event consuming` that can be used in the Presentation Layer. - -```python -class JoinMeetingCommandHandler(cqrs.RequestHandler[JoinMeetingCommand, None]): - def __init__(self): - self._events = [] - - @property - def events(self): - return self._events - - async def handle(self, request: JoinMeetingCommand) -> None: - STORAGE[request.meeting_id].append(request.user_id) - self._events.append( - UserJoined(user_id=request.user_id, meeting_id=request.meeting_id), - ) - print(f"User {request.user_id} joined meeting {request.meeting_id}") - - -class UserJoinedEventHandler(cqrs.EventHandler[UserJoined]): - async def handle(self, event: UserJoined) -> None: - print(f"Handle user {event.user_id} joined meeting {event.meeting_id} event") -``` - -A complete example can be found in -the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/domain_event_handler.py) - -### Parallel Event Processing - -Both `RequestMediator` and `StreamingRequestMediator` support parallel processing of domain events. You can control -the number of event handlers that run simultaneously using the `max_concurrent_event_handlers` parameter. - -This feature is especially useful when: -- Multiple event handlers need to process events independently -- You want to improve performance by processing events concurrently -- You need to limit resource consumption by controlling concurrency - -**Configuration:** - -```python -from cqrs.requests import bootstrap - -mediator = bootstrap.bootstrap_streaming( - di_container=container, - commands_mapper=commands_mapper, - domain_events_mapper=domain_events_mapper, - message_broker=broker, - max_concurrent_event_handlers=3, # Process up to 3 events in parallel - concurrent_event_handle_enable=True, # Enable parallel processing -) -``` - -> [!TIP] -> - Set `max_concurrent_event_handlers` to limit the number of simultaneously running event handlers -> - Set `concurrent_event_handle_enable=False` to disable parallel processing and process events sequentially -> - The default value for `max_concurrent_event_handlers` is `10` for `StreamingRequestMediator` and `1` for `RequestMediator` - ## Producing Notification Events During the handling of a command, `cqrs.NotificationEvent` events may be generated and then sent to the broker. @@ -839,7 +843,7 @@ async with session_factory() as session: ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/cqrs/blob/master/examples/kafka_outboxed_event_producing.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/kafka_outboxed_event_producing.py) ## Transaction log tailing @@ -853,126 +857,66 @@ The current version of the python-cqrs package does not support the implementati > which allows you to produce all newly created events within the Outbox storage directly to the corresponding topic in > Kafka (or any other broker). -## DI container - -Use the following example to set up dependency injection in your command, query and event handlers. This will make -dependency management simpler. - -The package supports two DI container libraries: +## Event Handlers -### di library +Event handlers are designed to process `Notification` and `ECST` events that are consumed from the broker. +To configure event handling, you need to implement a broker consumer on the side of your application. +Below is an example of `Kafka event consuming` that can be used in the Presentation Layer. ```python -import di -... +class JoinMeetingCommandHandler(cqrs.RequestHandler[JoinMeetingCommand, None]): + def __init__(self): + self._events = [] -def setup_di() -> di.Container: - """ - Binds implementations to dependencies - """ - container = di.Container() - container.bind( - di.bind_by_type( - dependent.Dependent(cqrs.SqlAlchemyOutboxedEventRepository, scope="request"), - cqrs.OutboxedEventRepository - ) - ) - container.bind( - di.bind_by_type( - dependent.Dependent(MeetingAPIImplementaion, scope="request"), - MeetingAPIProtocol + @property + def events(self): + return self._events + + async def handle(self, request: JoinMeetingCommand) -> None: + STORAGE[request.meeting_id].append(request.user_id) + self._events.append( + UserJoined(user_id=request.user_id, meeting_id=request.meeting_id), ) - ) - return container + print(f"User {request.user_id} joined meeting {request.meeting_id}") + + +class UserJoinedEventHandler(cqrs.EventHandler[UserJoined]): + async def handle(self, event: UserJoined) -> None: + print(f"Handle user {event.user_id} joined meeting {event.meeting_id} event") ``` A complete example can be found in -the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/dependency_injection.py) +the [documentation](https://github.com/vadikko2/python-cqrs/blob/master/examples/domain_event_handler.py) -### dependency-injector library +### Parallel Event Processing -The package also supports [dependency-injector](https://github.com/ets-labs/python-dependency-injector) library. -You can use `DependencyInjectorCQRSContainer` adapter to integrate dependency-injector containers with python-cqrs. +Both `RequestMediator` and `StreamingRequestMediator` support parallel processing of domain events. You can control +the number of event handlers that run simultaneously using the `max_concurrent_event_handlers` parameter. -```python -from dependency_injector import containers, providers -from cqrs.container.dependency_injector import DependencyInjectorCQRSContainer +This feature is especially useful when: +- Multiple event handlers need to process events independently +- You want to improve performance by processing events concurrently +- You need to limit resource consumption by controlling concurrency -class ApplicationContainer(containers.DeclarativeContainer): - # Define your providers - service = providers.Factory(ServiceImplementation) +**Configuration:** -# Create CQRS container adapter -cqrs_container = DependencyInjectorCQRSContainer(ApplicationContainer()) +```python +from cqrs.requests import bootstrap -# Use with bootstrap -mediator = bootstrap.bootstrap( - di_container=cqrs_container, +mediator = bootstrap.bootstrap_streaming( + di_container=container, commands_mapper=commands_mapper, - ... + domain_events_mapper=domain_events_mapper, + message_broker=broker, + max_concurrent_event_handlers=3, # Process up to 3 events in parallel + concurrent_event_handle_enable=True, # Enable parallel processing ) ``` -Complete examples can be found in: -- [Simple example](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injector_integration_simple_example.py) -- [Practical example with FastAPI](https://github.com/vadikko2/cqrs/blob/master/examples/dependency_injector_integration_practical_example.py) - -## Mapping - -To bind commands, queries and events with specific handlers, you can use the registries `EventMap` and `RequestMap`. - -```python -from cqrs import requests, events - -from app import commands, command_handlers -from app import queries, query_handlers -from app import events as event_models, event_handlers - - -def init_commands(mapper: requests.RequestMap) -> None: - mapper.bind(commands.JoinMeetingCommand, command_handlers.JoinMeetingCommandHandler) - -def init_queries(mapper: requests.RequestMap) -> None: - mapper.bind(queries.ReadMeetingQuery, query_handlers.ReadMeetingQueryHandler) - -def init_events(mapper: events.EventMap) -> None: - mapper.bind(events.NotificationEvent[events_models.NotificationMeetingRoomClosed], event_handlers.MeetingRoomClosedNotificationHandler) - mapper.bind(events.NotificationEvent[event_models.ECSTMeetingRoomClosed], event_handlers.UpdateMeetingRoomReadModelHandler) -``` - -## Bootstrap - -The `python-cqrs` package implements a set of bootstrap utilities designed to simplify the initial configuration of an -application. - -```python -import functools - -from cqrs.events import bootstrap as event_bootstrap -from cqrs.requests import bootstrap as request_bootstrap - -from app import dependencies, mapping, orm - - -@functools.lru_cache -def mediator_factory(): - return request_bootstrap.bootstrap( - di_container=dependencies.setup_di(), - commands_mapper=mapping.init_commands, - queries_mapper=mapping.init_queries, - domain_events_mapper=mapping.init_events, - on_startup=[orm.init_store_event_mapper], - ) - - -@functools.lru_cache -def event_mediator_factory(): - return event_bootstrap.bootstrap( - di_container=dependencies.setup_di(), - events_mapper=mapping.init_events, - on_startup=[orm.init_store_event_mapper], - ) -``` +> [!TIP] +> - Set `max_concurrent_event_handlers` to limit the number of simultaneously running event handlers +> - Set `concurrent_event_handle_enable=False` to disable parallel processing and process events sequentially +> - The default value for `max_concurrent_event_handlers` is `10` for `StreamingRequestMediator` and `1` for `RequestMediator` ## Integration with presentation layers