Merge pull request #73 from Zac-HD/hypothesis-determinism

Deterministic scheduling via Hypothesis

Author: pquentin

newsfragments/73.feature.rst

@@ -0,0 +1,3 @@
+pytest-trio now makes the Trio scheduler deterministic while running
+inside a Hypothesis test.  Hopefully you won't see any change, but if
+you had scheduler-dependent bugs Hypothesis will be more effective now.

pytest_trio/_tests/test_fixture_ordering.py

@@ -171,7 +171,7 @@ async def crash_late_agen():
             raise RuntimeError("crash_late_agen".upper())
 
         async def crash(when, token):
-            with trio.open_cancel_scope(shield=True):
+            with trio.CancelScope(shield=True):
                 await trio.sleep(when)
                 raise RuntimeError(token.upper())
 

pytest_trio/_tests/test_hypothesis_interaction.py

@@ -1,6 +1,13 @@
 import pytest
+import trio
+from trio.tests.test_scheduler_determinism import (
+    scheduler_trace, test_the_trio_scheduler_is_not_deterministic,
+    test_the_trio_scheduler_is_deterministic_if_seeded
+)
 from hypothesis import given, settings, strategies as st
 
+from pytest_trio.plugin import _trio_test_runner_factory
+
 # deadline=None avoids unpredictable warnings/errors when CI happens to be
 # slow (example: https://travis-ci.org/python-trio/pytest-trio/jobs/406738296)
 # max_examples=5 speeds things up a bit
@@ -28,3 +35,23 @@ async def test_mark_outer(n):
 async def test_mark_and_parametrize(x, y):
     assert x is None
     assert y in (1, 2)
+
+
+def test_the_trio_scheduler_is_deterministic_under_hypothesis():
+    traces = []
+
+    @our_settings
+    @given(st.integers())
+    @pytest.mark.trio
+    async def inner(_):
+        traces.append(await scheduler_trace())
+
+    # The pytest.mark.trio doesn't do it's magic thing to
+    # inner functions, so we invoke it explicitly here.
+    inner.hypothesis.inner_test = _trio_test_runner_factory(
+        None, inner.hypothesis.inner_test
+    )
+    inner()  # Tada, now it's a sync function!
+
+    assert len(traces) >= 5
+    assert len(set(traces)) == 1

pytest_trio/plugin.py

@@ -22,6 +22,20 @@
     # Ordered dict (and **kwargs) not available with Python<3.6
     ORDERED_DICTS = False
 
+try:
+    from hypothesis import register_random
+except ImportError:  # pragma: no cover
+    pass
+else:
+    # On recent versions of Hypothesis, make the Trio scheduler deterministic
+    # even though it uses a module-scoped Random instance.  This works
+    # regardless of whether or not the random_module strategy is used.
+    register_random(trio._core._run._r)
+    # We also have to enable determinism, which is disabled by default
+    # due to a small performance impact - but fine to enable in testing.
+    # See https://github.com/python-trio/trio/pull/890/ for details.
+    trio._core._run._ALLOW_DETERMINISTIC_SCHEDULING = True
+
 
 def pytest_addoption(parser):
     parser.addini(
@@ -225,7 +239,7 @@ async def run(self, test_ctx, contextvars_ctx):
                 # should cancel them.
                 assert not self.user_done_events
                 func_value = None
-                with trio.open_cancel_scope() as cancel_scope:
+                with trio.CancelScope() as cancel_scope:
                     test_ctx.test_cancel_scope = cancel_scope
                     assert not test_ctx.crashed
                     await self._func(**resolved_kwargs)
@@ -270,7 +284,7 @@ async def run(self, test_ctx, contextvars_ctx):
             except BaseException as exc:
                 assert isinstance(exc, trio.Cancelled)
                 test_ctx.crash(None)
-                with trio.open_cancel_scope(shield=True):
+                with trio.CancelScope(shield=True):
                     for event in self.user_done_events:
                         await event.wait()
 
@@ -345,7 +359,8 @@ def pytest_runtest_call(item):
             item.obj.hypothesis.inner_test = _trio_test_runner_factory(
                 item, item.obj.hypothesis.inner_test
             )
-        elif getattr(item.obj, 'is_hypothesis_test', False):
+        elif getattr(item.obj, 'is_hypothesis_test',
+                     False):  # pragma: no cover
             pytest.fail(
                 'test function `%r` is using Hypothesis, but pytest-trio '
                 'only works with Hypothesis 3.64.0 or later.' % item

setup.py

@@ -16,7 +16,7 @@
     packages=find_packages(),
     entry_points={'pytest11': ['trio = pytest_trio.plugin']},
     install_requires=[
-        "trio",
+        "trio >= 0.11",
         "async_generator >= 1.9",
         # For node.get_closest_marker
         "pytest >= 3.6"