feat: aggregate expect.soft() failures per test (#312)

* feat: aggregate expect.soft() failures per test

Adds an autouse function-scoped fixture in both pytest-playwright and
pytest-playwright-asyncio that wraps each test in a soft-assertion
collection scope. At end of test, collected failures are re-raised:
zero → noop, one → re-raise, multiple → BaseExceptionGroup.

Requires playwright-python with the soft-assertions hook
(microsoft/playwright-python#1272).

* refactor: lift BaseExceptionGroup import to module top

* feat: report soft assertion failures in call phase

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* chore: skip tests requiring Playwright 1.60 and bump EsrpRelease to v11

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Skn0tt

.azure-pipelines/publish.yml

@@ -55,7 +55,7 @@ extends:
             targetPath: $(Build.ArtifactStagingDirectory)/esrp-build
         steps:
           - checkout: none
-          - task: EsrpRelease@10
+          - task: EsrpRelease@11
             inputs:
               connectedservicename: 'Playwright-ESRP-PME'
               usemanagedidentity: true

pytest-playwright-asyncio/pytest_playwright_asyncio/pytest_playwright.py

@@ -37,6 +37,16 @@
 )
 
 import pytest
+
+if sys.version_info >= (3, 11):
+    _BaseExceptionGroup = BaseExceptionGroup  # noqa: F821
+else:
+    from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup
+
+try:
+    from playwright._impl._assertions import _soft_scope
+except ImportError:
+    _soft_scope = None
 from playwright.async_api import (
     Browser,
     BrowserContext,
@@ -89,6 +99,30 @@ def delete_output_dir(pytestconfig: Any) -> None:
                 shutil.rmtree(entry)
 
 
+@pytest.hookimpl(wrapper=True, tryfirst=True)
+def pytest_runtest_call(item: Any) -> Generator[None, Any, None]:
+    if _soft_scope is None:
+        yield
+        return
+    hard_failure: Optional[BaseException] = None
+    with _soft_scope() as errors:
+        try:
+            yield
+        except BaseException as exc:
+            hard_failure = exc
+    if not errors:
+        if hard_failure is not None:
+            raise hard_failure
+        return
+    if hard_failure is not None:
+        raise _BaseExceptionGroup(
+            "Test and soft assertion failures", [hard_failure, *errors]
+        )
+    if len(errors) == 1:
+        raise errors[0]
+    raise _BaseExceptionGroup("Soft assertion failures", errors)
+
+
 def pytest_generate_tests(metafunc: Any) -> None:
     if "browser_name" in metafunc.fixturenames:
         browsers = metafunc.config.option.browser or ["chromium"]

pytest-playwright/pytest_playwright/pytest_playwright.py

@@ -35,6 +35,16 @@
 )
 
 import pytest
+
+if sys.version_info >= (3, 11):
+    _BaseExceptionGroup = BaseExceptionGroup  # noqa: F821
+else:
+    from exceptiongroup import BaseExceptionGroup as _BaseExceptionGroup
+
+try:
+    from playwright._impl._assertions import _soft_scope
+except ImportError:
+    _soft_scope = None
 from playwright.sync_api import (
     Browser,
     BrowserContext,
@@ -86,6 +96,30 @@ def delete_output_dir(pytestconfig: Any) -> None:
                 shutil.rmtree(entry)
 
 
+@pytest.hookimpl(wrapper=True, tryfirst=True)
+def pytest_runtest_call(item: Any) -> Generator[None, Any, None]:
+    if _soft_scope is None:
+        yield
+        return
+    hard_failure: Optional[BaseException] = None
+    with _soft_scope() as errors:
+        try:
+            yield
+        except BaseException as exc:
+            hard_failure = exc
+    if not errors:
+        if hard_failure is not None:
+            raise hard_failure
+        return
+    if hard_failure is not None:
+        raise _BaseExceptionGroup(
+            "Test and soft assertion failures", [hard_failure, *errors]
+        )
+    if len(errors) == 1:
+        raise errors[0]
+    raise _BaseExceptionGroup("Soft assertion failures", errors)
+
+
 def pytest_generate_tests(metafunc: Any) -> None:
     if "browser_name" in metafunc.fixturenames:
         browsers = metafunc.config.option.browser or ["chromium"]

tests/test_asyncio.py

@@ -1045,6 +1045,7 @@ def test_with_page(page):
     ]
 
 
+@pytest.mark.skip(reason="requires 1.60")
 def test_connect_options_should_work(testdir: pytest.Testdir) -> None:
     server_process = None
     try:
@@ -1091,3 +1092,83 @@ async def test_connect_options(page):
         else:
             os.kill(server_process.pid, signal.SIGINT)
         server_process.wait()
+
+
+def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("goodbye")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    assert any("goodbye" in line for line in result.outlines)
+
+
+@pytest.mark.skip(reason="requires 1.60")
+def test_soft_assertion_multiple_failures_exception_group(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("first")
+            await expect.soft(page.locator("div")).to_have_text("second")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "first" in out and "second" in out
+
+
+@pytest.mark.skip(reason="requires 1.60")
+def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("hello")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(passed=1)
+
+
+@pytest.mark.skip(reason="requires 1.60")
+def test_soft_assertion_does_not_shadow_body_failure(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        from playwright.async_api import expect
+
+        @pytest.mark.asyncio
+        async def test_soft(page):
+            await page.set_content("<div>hello</div>")
+            await expect.soft(page.locator("div")).to_have_text("soft-fail")
+            raise RuntimeError("body-fail")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    # Body and soft assertion failures are grouped in call phase.
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "body-fail" in out
+    assert "soft-fail" in out

tests/test_sync.py

@@ -1035,6 +1035,7 @@ def test_with_page(page):
     ]
 
 
+@pytest.mark.skip(reason="requires 1.60")
 def test_connect_options_should_work(testdir: pytest.Testdir) -> None:
     server_process = None
     try:
@@ -1077,3 +1078,75 @@ def test_connect_options(page):
         else:
             os.kill(server_process.pid, signal.SIGINT)
         server_process.wait()
+
+
+def test_soft_assertion_single_failure(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("goodbye")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    assert any("goodbye" in line for line in result.outlines)
+
+
+@pytest.mark.skip(reason="requires 1.60")
+def test_soft_assertion_multiple_failures_exception_group(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("first")
+            expect.soft(page.locator("div")).to_have_text("second")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "first" in out and "second" in out
+
+
+@pytest.mark.skip(reason="requires 1.60")
+def test_soft_assertion_passes_when_all_match(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("hello")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    result.assert_outcomes(passed=1)
+
+
+@pytest.mark.skip(reason="requires 1.60")
+def test_soft_assertion_does_not_shadow_body_failure(
+    testdir: pytest.Testdir,
+) -> None:
+    testdir.makepyfile(
+        """
+        from playwright.sync_api import expect
+
+        def test_soft(page):
+            page.set_content("<div>hello</div>")
+            expect.soft(page.locator("div")).to_have_text("soft-fail")
+            raise RuntimeError("body-fail")
+    """
+    )
+    result = testdir.runpytest("--browser", "chromium")
+    # Body and soft assertion failures are grouped in call phase.
+    result.assert_outcomes(failed=1)
+    out = "\n".join(result.outlines)
+    assert "body-fail" in out
+    assert "soft-fail" in out