diff --git a/playwright/_impl/_assertions.py b/playwright/_impl/_assertions.py index aea37d35c..e781224d3 100644 --- a/playwright/_impl/_assertions.py +++ b/playwright/_impl/_assertions.py @@ -13,7 +13,8 @@ # limitations under the License. import collections.abc -from typing import Any, List, Optional, Pattern, Sequence, Union +from contextlib import contextmanager +from typing import Any, Iterator, List, Optional, Pattern, Sequence, Union from urllib.parse import urljoin from playwright._impl._api_structures import ( @@ -30,6 +31,33 @@ from playwright._impl._page import Page from playwright._impl._str_utils import escape_regex_flags +_soft_errors: Optional[List[AssertionError]] = None + + +@contextmanager +def _soft_scope() -> Iterator[List[AssertionError]]: + global _soft_errors + assert _soft_errors is None, "nested soft assertion scopes are not supported" + _soft_errors = [] + try: + yield _soft_errors + finally: + _soft_errors = None + + +def _record_soft_or_raise(error: AssertionError, is_soft: bool) -> None: + __tracebackhide__ = True + if is_soft: + if _soft_errors is None: + raise RuntimeError( + "expect.soft(...) requires pytest-playwright>=0.7.3 " + "(or pytest-playwright-asyncio>=0.7.3). Upgrade the plugin, " + "or use a regular expect(...) assertion." + ) + _soft_errors.append(error) + return + raise error + class AssertionsBase: def __init__( @@ -38,6 +66,7 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + is_soft: bool = False, ) -> None: self._actual_locator = locator self._loop = locator._loop @@ -45,6 +74,7 @@ def __init__( self._timeout = timeout self._is_not = is_not self._custom_message = message + self._is_soft = is_soft async def _call_expect( self, expression: str, expect_options: FrameExpectOptions, title: Optional[str] @@ -82,8 +112,11 @@ async def _expect_impl( ) error_message = result.get("errorMessage") error_message = f"\n{error_message}" if error_message else "" - raise AssertionError( - f"{out_message}\nActual value: {actual}{error_message} {format_call_log(result.get('log'))}" + _record_soft_or_raise( + AssertionError( + f"{out_message}\nActual value: {actual}{error_message} {format_call_log(result.get('log'))}" + ), + self._is_soft, ) @@ -94,8 +127,9 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + is_soft: bool = False, ) -> None: - super().__init__(page.locator(":root"), timeout, is_not, message) + super().__init__(page.locator(":root"), timeout, is_not, message, is_soft) self._actual_page = page async def _call_expect( @@ -109,7 +143,11 @@ async def _call_expect( @property def _not(self) -> "PageAssertions": return PageAssertions( - self._actual_page, self._timeout, not self._is_not, self._custom_message + self._actual_page, + self._timeout, + not self._is_not, + self._custom_message, + self._is_soft, ) async def to_have_title( @@ -169,8 +207,9 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + is_soft: bool = False, ) -> None: - super().__init__(locator, timeout, is_not, message) + super().__init__(locator, timeout, is_not, message, is_soft) self._actual_locator = locator async def _call_expect( @@ -182,7 +221,11 @@ async def _call_expect( @property def _not(self) -> "LocatorAssertions": return LocatorAssertions( - self._actual_locator, self._timeout, not self._is_not, self._custom_message + self._actual_locator, + self._timeout, + not self._is_not, + self._custom_message, + self._is_soft, ) async def to_contain_text( @@ -944,6 +987,7 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + is_soft: bool = False, ) -> None: self._loop = response._loop self._dispatcher_fiber = response._dispatcher_fiber @@ -951,11 +995,16 @@ def __init__( self._is_not = is_not self._actual = response self._custom_message = message + self._is_soft = is_soft @property def _not(self) -> "APIResponseAssertions": return APIResponseAssertions( - self._actual, self._timeout, not self._is_not, self._custom_message + self._actual, + self._timeout, + not self._is_not, + self._custom_message, + self._is_soft, ) async def to_be_ok( @@ -976,7 +1025,7 @@ async def to_be_ok( if text is not None: out_message += f"\n Response Text:\n{text[:1000]}" - raise AssertionError(out_message) + _record_soft_or_raise(AssertionError(out_message), self._is_soft) async def not_to_be_ok(self) -> None: __tracebackhide__ = True diff --git a/playwright/async_api/__init__.py b/playwright/async_api/__init__.py index 6508994c3..d01b711f0 100644 --- a/playwright/async_api/__init__.py +++ b/playwright/async_api/__init__.py @@ -129,19 +129,64 @@ def __call__( def __call__( self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None + ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: + return self._dispatch(actual, message, is_soft=False) + + @overload + def soft(self, actual: Page, message: Optional[str] = None) -> PageAssertions: ... + + @overload + def soft( + self, actual: Locator, message: Optional[str] = None + ) -> LocatorAssertions: ... + + @overload + def soft( + self, actual: APIResponse, message: Optional[str] = None + ) -> APIResponseAssertions: ... + + def soft( + self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None + ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: + """ + Creates a [soft assertion](https://playwright.dev/python/docs/test-assertions#soft-assertions). + Failing soft assertions do not abort test execution, but mark the test + as failed. Multiple failures from the same test are surfaced together + at the end of the test. + """ + return self._dispatch(actual, message, is_soft=True) + + def _dispatch( + self, + actual: Union[Page, Locator, APIResponse], + message: Optional[str], + is_soft: bool, ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: if isinstance(actual, Page): return PageAssertions( - PageAssertionsImpl(actual._impl_obj, self._timeout, message=message) + PageAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + is_soft=is_soft, + ) ) elif isinstance(actual, Locator): return LocatorAssertions( - LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message) + LocatorAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + is_soft=is_soft, + ) ) elif isinstance(actual, APIResponse): return APIResponseAssertions( APIResponseAssertionsImpl( - actual._impl_obj, self._timeout, message=message + actual._impl_obj, + self._timeout, + message=message, + is_soft=is_soft, ) ) raise ValueError(f"Unsupported type: {type(actual)}") diff --git a/playwright/sync_api/__init__.py b/playwright/sync_api/__init__.py index 6015bc1be..0f6592bc0 100644 --- a/playwright/sync_api/__init__.py +++ b/playwright/sync_api/__init__.py @@ -129,19 +129,64 @@ def __call__( def __call__( self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None + ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: + return self._dispatch(actual, message, is_soft=False) + + @overload + def soft(self, actual: Page, message: Optional[str] = None) -> PageAssertions: ... + + @overload + def soft( + self, actual: Locator, message: Optional[str] = None + ) -> LocatorAssertions: ... + + @overload + def soft( + self, actual: APIResponse, message: Optional[str] = None + ) -> APIResponseAssertions: ... + + def soft( + self, actual: Union[Page, Locator, APIResponse], message: Optional[str] = None + ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: + """ + Creates a [soft assertion](https://playwright.dev/python/docs/test-assertions#soft-assertions). + Failing soft assertions do not abort test execution, but mark the test + as failed. Multiple failures from the same test are surfaced together + at the end of the test. + """ + return self._dispatch(actual, message, is_soft=True) + + def _dispatch( + self, + actual: Union[Page, Locator, APIResponse], + message: Optional[str], + is_soft: bool, ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: if isinstance(actual, Page): return PageAssertions( - PageAssertionsImpl(actual._impl_obj, self._timeout, message=message) + PageAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + is_soft=is_soft, + ) ) elif isinstance(actual, Locator): return LocatorAssertions( - LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message) + LocatorAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + is_soft=is_soft, + ) ) elif isinstance(actual, APIResponse): return APIResponseAssertions( APIResponseAssertionsImpl( - actual._impl_obj, self._timeout, message=message + actual._impl_obj, + self._timeout, + message=message, + is_soft=is_soft, ) ) raise ValueError(f"Unsupported type: {type(actual)}") diff --git a/tests/async/test_assertions.py b/tests/async/test_assertions.py index 49e3c3e7f..0d974543a 100644 --- a/tests/async/test_assertions.py +++ b/tests/async/test_assertions.py @@ -1132,3 +1132,50 @@ async def test_to_have_role(page: Page) -> None: with pytest.raises(Error) as excinfo: await expect(page.locator("div")).to_have_role(re.compile(r"button|checkbox")) # type: ignore assert '"role" argument in to_have_role must be a string' in str(excinfo.value) + + +async def test_soft_outside_scope_raises_runtime_error( + page: Page, server: Server +) -> None: + await page.set_content("
hello
") + with pytest.raises(RuntimeError, match="pytest-playwright"): + await expect.soft(page.locator("div")).to_have_text("nope", timeout=500) + + +async def test_soft_inside_scope_collects_failures(page: Page, server: Server) -> None: + from playwright._impl._assertions import _soft_scope + + await page.goto(server.EMPTY_PAGE) + await page.set_content("actual
hello
") + + with _soft_scope() as errors: + # should collect, not raise + await expect.soft(page).to_have_title("expected", timeout=500) + await expect.soft(page.locator("div")).to_have_text("goodbye", timeout=500) + # passing soft should not affect the collector + await expect.soft(page.locator("div")).to_have_text("hello") + # nested .not_ should still be soft + await expect.soft(page.locator("div")).not_to_have_text("hello", timeout=500) + + assert len(errors) == 3 + assert all(isinstance(e, AssertionError) for e in errors) + + +async def test_soft_does_not_leak_between_scopes(page: Page, server: Server) -> None: + from playwright._impl._assertions import _soft_scope + + await page.goto(server.EMPTY_PAGE) + await page.set_content("actual") + + with _soft_scope() as errors_a: + await expect.soft(page).to_have_title("nope", timeout=500) + assert len(errors_a) == 1 + + with _soft_scope() as errors_b: + pass + assert errors_b == [] + + # After scope ends, soft assertions raise RuntimeError again. + await page.set_content("
hello
") + with pytest.raises(RuntimeError, match="pytest-playwright"): + await expect.soft(page.locator("div")).to_have_text("nope", timeout=500) diff --git a/tests/sync/test_assertions.py b/tests/sync/test_assertions.py index 0199f67df..a3b98ed1d 100644 --- a/tests/sync/test_assertions.py +++ b/tests/sync/test_assertions.py @@ -1067,3 +1067,25 @@ def test_to_have_role(page: Page) -> None: with pytest.raises(Error) as excinfo: expect(page.locator("div")).to_have_role(re.compile(r"button|checkbox")) # type: ignore assert '"role" argument in to_have_role must be a string' in str(excinfo.value) + + +def test_soft_outside_scope_raises_runtime_error(page: Page, server: Server) -> None: + page.set_content("
hello
") + with pytest.raises(RuntimeError, match="pytest-playwright"): + expect.soft(page.locator("div")).to_have_text("nope", timeout=500) + + +def test_soft_inside_scope_collects_failures(page: Page, server: Server) -> None: + from playwright._impl._assertions import _soft_scope + + page.goto(server.EMPTY_PAGE) + page.set_content("actual
hello
") + + with _soft_scope() as errors: + expect.soft(page).to_have_title("expected", timeout=500) + expect.soft(page.locator("div")).to_have_text("goodbye", timeout=500) + expect.soft(page.locator("div")).to_have_text("hello") + expect.soft(page.locator("div")).not_to_have_text("hello", timeout=500) + + assert len(errors) == 3 + assert all(isinstance(e, AssertionError) for e in errors)