diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..3bb710c9f1 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,18 @@ +RELEASE_TYPE: minor + +This release adds :ref:`observability `, a new feature in Hypothesis. + +Observability is intended for those who want to dig into the raw data from tests or strategies—including those building tools or research on top of Hypothesis. + +When observability is enabled, Hypothesis writes data about each test case to the ``.hypothesis/observed`` directory in an analysis-ready `jsonlines `_ format. This data includes a timing breakdown, coverage data, a representation of each argument, the result of any |assume| or |.filter| calls, and any other data we think is useful for analysis. See :ref:`observability ` for the full data format. + +Observability can be controlled in two ways: + +* via the new |settings.observability| argument, +* or via the new ``HYPOTHESIS_OBSERVABILITY`` environment variable. + +See :ref:`Configuring observability ` for details. + +If you use VSCode, we recommend the `Tyche `__ extension, a PBT-specific visualization tool designed for Hypothesis's observability interface. + +(Note that Hypothesis has had an implementation of observability for a number of months, which was marked as experimental. This release brings observability into our public API, and removes that experimental label.) diff --git a/hypothesis-python/docs/changelog.rst b/hypothesis-python/docs/changelog.rst index fda1280234..7fcec00e39 100644 --- a/hypothesis-python/docs/changelog.rst +++ b/hypothesis-python/docs/changelog.rst @@ -641,7 +641,7 @@ Fixes a bug with solver-based :ref:`alternative backends ` 6.137.0 - 2025-08-05 -------------------- -Add the |add_observability_callback|, |remove_observability_callback|, |with_observability_callback|, and |observability_enabled| methods to the :ref:`observability ` interface. The previous |TESTCASE_CALLBACKS| is deprecated. +Add the ``add_observability_callback``, ``remove_observability_callback``, ``with_observability_callback``, and ``observability_enabled`` methods to the :ref:`observability ` interface. The previous ``TESTCASE_CALLBACKS`` is deprecated. This release also adds better threading support to observability callbacks. An observability callback will now only be called for observations generated by the same thread. @@ -976,11 +976,11 @@ Further improve the performance of the constants-collection feature introduced i 6.135.3 - 2025-06-08 -------------------- -This release adds the experimental and unstable |OBSERVABILITY_CHOICES| option for :ref:`observability `. If set, the choice sequence is included in ``metadata.choice_nodes``, and choice sequence spans are included in ``metadata.choice_spans``. +This release adds the experimental and unstable ``OBSERVABILITY_CHOICES`` option for :ref:`observability `. If set, the choice sequence is included in ``metadata.choice_nodes``, and choice sequence spans are included in ``metadata.choice_spans``. These are relatively low-level implementation detail of Hypothesis, and are exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence and choice spans. -We are actively working towards a better interface for this. Feel free to use |OBSERVABILITY_CHOICES| to experiment, but don't rely on it yet! +We are actively working towards a better interface for this. Feel free to use ``OBSERVABILITY_CHOICES`` to experiment, but don't rely on it yet! .. _v6.135.2: @@ -1097,7 +1097,7 @@ This patch resolves a Pandas FutureWarning (:issue:`4400`) caused by indexing wi 6.131.29 - 2025-05-27 --------------------- -The observations passed to |TESTCASE_CALLBACKS| are now dataclasses, rather than dictionaries. The content written to ``.hypothesis/observed`` under ``HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY`` remains the same. +The observations passed to ``TESTCASE_CALLBACKS`` are now dataclasses, rather than dictionaries. The content written to ``.hypothesis/observed`` under ``HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY`` remains the same. .. _v6.131.28: diff --git a/hypothesis-python/docs/prolog.rst b/hypothesis-python/docs/prolog.rst index 99b6ae2cad..2bb4bb4304 100644 --- a/hypothesis-python/docs/prolog.rst +++ b/hypothesis-python/docs/prolog.rst @@ -27,6 +27,7 @@ .. |settings.suppress_health_check| replace:: :obj:`settings.suppress_health_check ` .. |settings.stateful_step_count| replace:: :obj:`settings.stateful_step_count ` .. |settings.backend| replace:: :obj:`settings.backend ` +.. |settings.observability| replace:: :obj:`settings.observability ` .. |~settings.max_examples| replace:: :obj:`~hypothesis.settings.max_examples` .. |~settings.database| replace:: :obj:`~hypothesis.settings.database` @@ -39,6 +40,7 @@ .. |~settings.suppress_health_check| replace:: :obj:`~hypothesis.settings.suppress_health_check` .. |~settings.stateful_step_count| replace:: :obj:`~hypothesis.settings.stateful_step_count` .. |~settings.backend| replace:: :obj:`~hypothesis.settings.backend` +.. |~settings.observability| replace:: :obj:`~hypothesis.settings.observability` .. |settings.register_profile| replace:: :func:`~hypothesis.settings.register_profile` .. |settings.get_profile| replace:: :func:`~hypothesis.settings.get_profile` @@ -66,6 +68,8 @@ .. |Verbosity.normal| replace:: :obj:`Verbosity.normal ` .. |Verbosity.quiet| replace:: :obj:`Verbosity.quiet ` +.. |ObservabilityConfig| replace:: :obj:`ObservabilityConfig ` + .. |HypothesisException| replace:: :obj:`HypothesisException ` .. |HypothesisDeprecationWarning| replace:: :obj:`HypothesisDeprecationWarning ` .. |NonInteractiveExampleWarning| replace:: :obj:`NonInteractiveExampleWarning ` @@ -145,12 +149,6 @@ .. |AVAILABLE_PROVIDERS| replace:: :data:`~hypothesis.internal.conjecture.providers.AVAILABLE_PROVIDERS` .. |run_conformance_test| replace:: :func:`~hypothesis.internal.conjecture.provider_conformance.run_conformance_test` -.. |add_observability_callback| replace:: :data:`~hypothesis.internal.observability.add_observability_callback` -.. |remove_observability_callback| replace:: :data:`~hypothesis.internal.observability.remove_observability_callback` -.. |with_observability_callback| replace:: :data:`~hypothesis.internal.observability.with_observability_callback` -.. |observability_enabled| replace:: :data:`~hypothesis.internal.observability.observability_enabled` -.. |TESTCASE_CALLBACKS| replace:: :data:`~hypothesis.internal.observability.TESTCASE_CALLBACKS` -.. |OBSERVABILITY_CHOICES| replace:: :data:`~hypothesis.internal.observability.OBSERVABILITY_CHOICES` .. |BUFFER_SIZE| replace:: :data:`~hypothesis.internal.conjecture.engine.BUFFER_SIZE` .. |MAX_SHRINKS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKS` .. |MAX_SHRINKING_SECONDS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKING_SECONDS` diff --git a/hypothesis-python/docs/reference/api.rst b/hypothesis-python/docs/reference/api.rst index a382a34ab5..fa88c708f1 100644 --- a/hypothesis-python/docs/reference/api.rst +++ b/hypothesis-python/docs/reference/api.rst @@ -174,6 +174,8 @@ Settings .. autoclass:: hypothesis.Verbosity :members: +.. autoclass:: hypothesis.ObservabilityConfig + .. autoclass:: hypothesis.HealthCheck :undoc-members: :inherited-members: diff --git a/hypothesis-python/docs/reference/integrations.rst b/hypothesis-python/docs/reference/integrations.rst index 0cc847a071..f59236e34f 100644 --- a/hypothesis-python/docs/reference/integrations.rst +++ b/hypothesis-python/docs/reference/integrations.rst @@ -75,17 +75,10 @@ If you're interested in similar questions, `drop me an email`_! Observability ------------- -.. note:: +.. tip:: The `Tyche `__ VSCode extension provides an in-editor UI for observability results generated by Hypothesis. If you want to *view* observability results, rather than programmatically consume or display them, we recommend using Tyche. -.. warning:: - - This feature is experimental, and could have breaking changes or even be removed - without notice. Try it out, let us know what you think, but don't rely on it - just yet! - - Motivation ~~~~~~~~~~ @@ -108,12 +101,16 @@ debuggers such as `rr `__ or `pytrace `, or by using the :pypi:`sqlite-utils` and :pypi:`datasette` libraries:: @@ -121,11 +118,6 @@ or by using the :pypi:`sqlite-utils` and :pypi:`datasette` libraries:: sqlite-utils insert testcases.db testcases .hypothesis/observed/*_testcases.jsonl --nl --flatten datasette serve testcases.db -If you are experiencing a significant slow-down, you can try setting -``HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_NOCOVER`` instead; this will disable coverage information -collection. This should not be necessary on Python 3.12 or later, where coverage collection is very fast. - - Collecting more information ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -181,7 +173,7 @@ While the observability format is agnostic to the property-based testing library Choices metadata ++++++++++++++++ -These additional metadata elements are included in ``metadata`` (as e.g. ``metadata["choice_nodes"]`` or ``metadata["choice_spans"]``), if and only if |OBSERVABILITY_CHOICES| is set. +These additional metadata elements are included in ``metadata`` (as e.g. ``metadata["choice_nodes"]`` or ``metadata["choice_spans"]``), if and only if observability is configured to include choices (see |ObservabilityConfig|). .. jsonschema:: ./schema_metadata_choices.json :hide_key: /additionalProperties, /type diff --git a/hypothesis-python/docs/reference/internals.rst b/hypothesis-python/docs/reference/internals.rst index 01b76227f9..e4aea5ac3d 100644 --- a/hypothesis-python/docs/reference/internals.rst +++ b/hypothesis-python/docs/reference/internals.rst @@ -27,18 +27,6 @@ Alternative backends .. autoclass:: hypothesis.errors.BackendCannotProceed .. autoclass:: hypothesis.internal.intervalsets.IntervalSet -Observability -------------- - -.. autofunction:: hypothesis.internal.observability.add_observability_callback -.. autofunction:: hypothesis.internal.observability.remove_observability_callback -.. autofunction:: hypothesis.internal.observability.with_observability_callback -.. autofunction:: hypothesis.internal.observability.observability_enabled - -.. autodata:: hypothesis.internal.observability.TESTCASE_CALLBACKS -.. autodata:: hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE -.. autodata:: hypothesis.internal.observability.OBSERVABILITY_CHOICES - Engine constants ---------------- diff --git a/hypothesis-python/docs/reference/schema_metadata_choices.json b/hypothesis-python/docs/reference/schema_metadata_choices.json index 62ce2b9a29..758a7f234b 100644 --- a/hypothesis-python/docs/reference/schema_metadata_choices.json +++ b/hypothesis-python/docs/reference/schema_metadata_choices.json @@ -3,7 +3,7 @@ "properties": { "choice_nodes": { "type": ["array", "null"], - "description": ".. warning::\n\n EXPERIMENTAL AND UNSTABLE. This attribute may change format or disappear without warning.\n\nThe sequence of choices made during this test case. This includes the choice value, as well as its constraints and whether it was forced or not.\n\nOnly present if |OBSERVABILITY_CHOICES| is ``True``.\n\n.. note::\n\n The choice sequence is a relatively low-level implementation detail of Hypothesis, and is exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence.", + "description": ".. warning::\n\n EXPERIMENTAL AND UNSTABLE. This attribute may change format without warning.\n\nThe sequence of choices made during this test case. This includes the choice value, as well as its constraints and whether it was forced or not.\n\n.. note::\n\n Only present if observability is configured to include choices (see |ObservabilityConfig|).\n\n.. note::\n\n The choice sequence is a relatively low-level implementation detail of Hypothesis, and is exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence.", "items": { "type": "object", "properties": { @@ -31,7 +31,7 @@ "choice_spans": { "type": "array", "items": {"type": "array"}, - "description": ".. warning::\n\n EXPERIMENTAL AND UNSTABLE. This attribute may change format or disappear without warning.\n\nThe semantically-meaningful spans of the choice sequence of this test case.\n\nEach span has the format ``[label, start, end, discarded]``, where:\n\n* ``label`` is an opaque integer-value string shared by all spans drawn from a particular strategy.\n* ``start`` and ``end`` are indices into the choice sequence for this span, such that ``choices[start:end]`` are the corresponding choices.\n* ``discarded`` is a boolean indicating whether this span was discarded (see |PrimitiveProvider.span_end|).\n\nOnly present if |OBSERVABILITY_CHOICES| is ``True``.\n\n.. note::\n\n Spans are a relatively low-level implementation detail of Hypothesis, and are exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| (and particularly |PrimitiveProvider.span_start| and |PrimitiveProvider.span_end|) for more details about spans." + "description": ".. warning::\n\n EXPERIMENTAL AND UNSTABLE. This attribute may change format without warning.\n\nThe semantically-meaningful spans of the choice sequence of this test case.\n\nEach span has the format ``[label, start, end, discarded]``, where:\n\n* ``label`` is an opaque integer-value string shared by all spans drawn from a particular strategy.\n* ``start`` and ``end`` are indices into the choice sequence for this span, such that ``choices[start:end]`` are the corresponding choices.\n* ``discarded`` is a boolean indicating whether this span was discarded (see |PrimitiveProvider.span_end|).\n\n.. note::\n\n Only present if observability is configured to include choices (see |ObservabilityConfig|).\n\n.. note::\n\n Spans are a relatively low-level implementation detail of Hypothesis, and are exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| (and particularly |PrimitiveProvider.span_start| and |PrimitiveProvider.span_end|) for more details about spans." } }, "required": ["traceback", "reproduction_decorator", "predicates", "backend", "sys.argv", "os.getpid()", "imported_at", "data_status", "interesting_origin", "choice_nodes", "choice_spans"], diff --git a/hypothesis-python/docs/reference/schema_observations.json b/hypothesis-python/docs/reference/schema_observations.json index 7f98592fb7..225ace3e9f 100644 --- a/hypothesis-python/docs/reference/schema_observations.json +++ b/hypothesis-python/docs/reference/schema_observations.json @@ -36,7 +36,7 @@ }, "coverage": { "type": ["object", "null"], - "description": "Mapping of filename to list of covered line numbers, if coverage information is available, or None if not. Hypothesis deliberately omits stdlib and site-packages code.", + "description": ".. warning::\n\n EXPERIMENTAL AND UNSTABLE. This attribute might change format without warning, as we figure out the right format to support both line and branch coverage.\n\n Mapping of filename to list of covered line numbers, if coverage information is available, or None if not. Hypothesis deliberately omits stdlib and site-packages code.\n\n.. note::\n\n Only present if observability is configured to include coverage (see |ObservabilityConfig|).", "additionalProperties": { "type": "array", "items": {"type": "integer", "minimum": 1}, diff --git a/hypothesis-python/src/hypothesis/__init__.py b/hypothesis-python/src/hypothesis/__init__.py index 0efe43985d..f459e3f26f 100644 --- a/hypothesis-python/src/hypothesis/__init__.py +++ b/hypothesis-python/src/hypothesis/__init__.py @@ -17,7 +17,12 @@ import _hypothesis_globals -from hypothesis._settings import HealthCheck, Phase, Verbosity, settings +from hypothesis._settings import ( + HealthCheck, + Phase, + Verbosity, + settings, +) from hypothesis.control import ( assume, currently_in_test_context, @@ -30,11 +35,13 @@ from hypothesis.entry_points import run from hypothesis.internal.detection import is_hypothesis_test from hypothesis.internal.entropy import register_random +from hypothesis.internal.observability import ObservabilityConfig from hypothesis.utils.conventions import infer from hypothesis.version import __version__, __version_info__ __all__ = [ "HealthCheck", + "ObservabilityConfig", "Phase", "Verbosity", "__version__", diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index 0d66eb6336..10abc6cbfa 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -29,10 +29,12 @@ TypeVar, ) -from hypothesis.errors import ( - InvalidArgument, -) +from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture.providers import AVAILABLE_PROVIDERS +from hypothesis.internal.observability import ( + ObservabilityConfig, + envvar_observability, +) from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.internal.validation import check_type, try_convert from hypothesis.utils.conventions import not_set @@ -57,6 +59,7 @@ "deadline", "print_blob", "backend", + "observability", ] @@ -533,6 +536,18 @@ def _validate_backend(backend: str) -> str: return backend +def _validate_observability(observability: Any) -> ObservabilityConfig | None: + check_type( + (bool, ObservabilityConfig, type(None)), observability, name="observability" + ) + + if isinstance(observability, ObservabilityConfig): + return observability + + assert observability in {True, False, None} + return ObservabilityConfig() if observability else None + + class settingsMeta(type): def __init__(cls, *args, **kwargs): super().__init__(*args, **kwargs) @@ -573,7 +588,8 @@ class settings(metaclass=settingsMeta): |~settings.max_examples|, |~settings.derandomize|, |~settings.database|, |~settings.verbosity|, |~settings.phases|, |~settings.stateful_step_count|, |~settings.report_multiple_bugs|, |~settings.suppress_health_check|, - |~settings.deadline|, |~settings.print_blob|, and |~settings.backend|. + |~settings.deadline|, |~settings.print_blob|, |~settings.backend|, and + |~settings.observability|. A settings object can be applied as a decorator to a test function, in which case that test function will use those settings. A test may only have one @@ -630,6 +646,7 @@ class settings(metaclass=settingsMeta): deadline=duration(milliseconds=200), print_blob=False, backend="hypothesis", + observability=False, ) ci = settings.register_profile( @@ -676,6 +693,7 @@ def __init__( deadline: int | float | datetime.timedelta | None = not_set, # type: ignore print_blob: bool = not_set, # type: ignore backend: str = not_set, # type: ignore + observability: bool | ObservabilityConfig | None = not_set, # type: ignore ) -> None: self._in_definition = True @@ -702,10 +720,12 @@ def __init__( if derandomize is not_set # type: ignore else _validate_choices("derandomize", derandomize, choices=[True, False]) ) + if database is not not_set: # type: ignore database = _validate_database(database) self._database = database self._cached_database = None + self._verbosity = ( self._fallback.verbosity # type: ignore if verbosity is not_set # type: ignore @@ -749,6 +769,15 @@ def __init__( else _validate_backend(backend) ) + if observability is not_set: # type: ignore + self._observability = ( + envvar_observability + if self._fallback is None + else self._fallback.observability + ) + else: + self._observability = _validate_observability(observability) + self._in_definition = False @property @@ -1030,6 +1059,37 @@ def backend(self): """ return self._backend + @property + def observability(self): + """ + Controls the :ref:`observability ` behavior of Hypothesis. + + Valid values are ``True``, ``False``, ``None``, or |ObservabilityConfig|. If + ``True`` is passed, observability is enabled, and if ``False`` or ``None`` + is passed, observability is disabled. + + If |ObservabilityConfig| is passed, observability will be configured based + on the options passed to |ObservabilityConfig|. + + For example: + + .. code-block:: python + + from hypothesis import settings, ObservabilityConfig + + # enables observability + settings(observability=True) + + # enables observability, with choice sequence data + settings(observability=ObservabilityConfig(choices=True)) + + By default, |settings.observability| is inherited from the + :ref:`HYPOTHESIS_OBSERVABILITY ` + environment variable. If that environment variable is also not set, + |settings.observability| defaults to ``False``. + """ + return self._observability + def __call__(self, test: T) -> T: """Make the settings object (self) an attribute of the test. @@ -1202,6 +1262,7 @@ def local_settings(s: settings) -> Generator[settings, None, None]: deadline=duration(milliseconds=200), print_blob=False, backend="hypothesis", + observability=not_set, # type: ignore ) settings.register_profile("default", default) settings.load_profile("default") diff --git a/hypothesis-python/src/hypothesis/control.py b/hypothesis-python/src/hypothesis/control.py index d0aee23d2f..e2392900ba 100644 --- a/hypothesis-python/src/hypothesis/control.py +++ b/hypothesis-python/src/hypothesis/control.py @@ -21,7 +21,6 @@ from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption from hypothesis.internal.compat import BaseExceptionGroup from hypothesis.internal.conjecture.data import ConjectureData -from hypothesis.internal.observability import observability_enabled from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.internal.validation import check_type from hypothesis.reporting import report, verbose_report @@ -68,9 +67,11 @@ def assume(condition: object) -> Literal[True]: since="2023-09-25", has_codemod=False, ) - if observability_enabled() or not condition: + + observability_enabled = settings().observability is not None + if observability_enabled or not condition: where = _calling_function_location("assume", inspect.currentframe()) - if observability_enabled() and currently_in_test_context(): + if observability_enabled and currently_in_test_context(): counts = current_build_context().data._observability_predicates[where] counts.update_count(condition=bool(condition)) if not condition: diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index d7d1b39e59..e27624d97a 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -70,7 +70,6 @@ Unsatisfiable, UnsatisfiedAssumption, ) -from hypothesis.internal import observability from hypothesis.internal.compat import ( PYPY, BaseExceptionGroup, @@ -103,9 +102,8 @@ from hypothesis.internal.observability import ( InfoObservation, InfoObservationType, - deliver_observation, + Observation, make_testcase, - observability_enabled, ) from hypothesis.internal.reflection import ( convert_positional_arguments, @@ -688,7 +686,7 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s ) empty_data.freeze() - if observability_enabled(): + if state.settings.observability is not None: tc = make_testcase( run_start=state._start_timestamp, property=state.test_identifier, @@ -696,8 +694,9 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s how_generated="explicit example", representation=state._string_repr, timing=state._timing_features, + observability=state.settings.observability, ) - deliver_observation(tc) + state._deliver_observation(tc) if fragments_reported: verbose_report(fragments_reported[0].replace("Falsifying", "Trying", 1)) @@ -963,7 +962,8 @@ def _should_trace(self): # NOTE: we explicitly support monkeypatching this. Keep the namespace # access intact. _trace_obs = ( - observability_enabled() and observability.OBSERVABILITY_COLLECT_COVERAGE + self.settings.observability is not None + and self.settings.observability.coverage ) _trace_failure = ( self.failed_normally @@ -996,7 +996,7 @@ def execute_once( self._string_repr = "" text_repr = None - if self.settings.deadline is None and not observability_enabled(): + if self.settings.deadline is None and self.settings.observability is None: @proxies(self.test) def test(*args, **kwargs): @@ -1086,7 +1086,7 @@ def run(data: ConjectureData) -> None: ) report(printer.getvalue()) - if observability_enabled(): + if self.settings.observability is not None: printer = RepresentationPrinter(context=context) printer.repr_call( test.__name__, @@ -1119,7 +1119,7 @@ def run(data: ConjectureData) -> None: if data._stateful_repr_parts is not None: self._string_repr = "\n".join(data._stateful_repr_parts) - if observability_enabled(): + if self.settings.observability is not None: printer = RepresentationPrinter(context=context) for name, value in data._observability_args.items(): if name.startswith("generate:Draw "): @@ -1320,7 +1320,7 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None: except BackendCannotProceed: data.events = {} - if observability_enabled(): + if self.settings.observability is not None: if runner := getattr(self, "_runner", None): phase = runner._current_phase else: # pragma: no cover # in case of messing with internals @@ -1356,8 +1356,9 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None: coverage=tractable_coverage_report(trace) or None, phase=phase, backend_metadata=data.provider.observe_test_case(), + observability=self.settings.observability, ) - deliver_observation(tc) + self._deliver_observation(tc) for msg in data.provider.observe_information_messages( lifetime="test_case" @@ -1365,10 +1366,18 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None: self._deliver_information_message(**msg) self._timing_features = {} + def _deliver_observation(self, observation: Observation) -> None: + # nocover because we currently guard all calls to this by checking if + # observability is enabled, but that might not always be true + if self.settings.observability is None: # pragma: no cover + return + for callback in self.settings.observability.callbacks: + callback(observation) + def _deliver_information_message( self, *, type: InfoObservationType, title: str, content: str | dict ) -> None: - deliver_observation( + self._deliver_observation( InfoObservation( type=type, run_start=self._start_timestamp, @@ -1404,7 +1413,7 @@ def run_engine(self): # on different inputs. runner.run() note_statistics(runner.statistics) - if observability_enabled(): + if self.settings.observability is not None: self._deliver_information_message( type="info", title="Hypothesis Statistics", @@ -1544,7 +1553,7 @@ def run_engine(self): raise NotImplementedError("This should be unreachable") finally: ran_example.freeze() - if observability_enabled(): + if self.settings.observability is not None: # log our observability line for the final failing example tc = make_testcase( run_start=self._start_timestamp, @@ -1558,8 +1567,9 @@ def run_engine(self): status="passed" if sys.exc_info()[0] else "failed", status_reason=str(origin or "unexpected/flaky pass"), metadata={"traceback": tb}, + observability=self.settings.observability, ) - deliver_observation(tc) + self._deliver_observation(tc) # Whether or not replay actually raised the exception again, we want # to print the reproduce_failure decorator for the failing example. @@ -2322,7 +2332,7 @@ def fuzz_one_input( status = Status.INTERESTING raise finally: - if observability_enabled(): + if state.settings.observability is not None: data.freeze() tc = make_testcase( run_start=state._start_timestamp, @@ -2335,8 +2345,9 @@ def fuzz_one_input( coverage=None, status=status, backend_metadata=data.provider.observe_test_case(), + observability=state.settings.observability, ) - deliver_observation(tc) + state._deliver_observation(tc) state._timing_features = {} assert isinstance(data.provider, BytestringProvider) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 8c6ea5c4f1..6774b11671 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -27,6 +27,7 @@ overload, ) +from hypothesis._settings import settings from hypothesis.errors import ( CannotProceedScopeT, ChoiceTooLarge, @@ -1170,7 +1171,6 @@ def draw( label: int | None = None, observe_as: str | None = None, ) -> "Ex": - from hypothesis.internal.observability import observability_enabled from hypothesis.strategies._internal.lazy import unwrap_strategies from hypothesis.strategies._internal.utils import to_jsonable @@ -1219,7 +1219,7 @@ def draw( f"while generating {key.removeprefix('generate:')!r} from {strategy!r}", ) raise - if observability_enabled(): + if settings().observability is not None: avoid = self.provider.avoid_realization self._observability_args[key] = to_jsonable(v, avoid_realization=avoid) return v diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 531797c7ab..2f18bccb37 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -8,6 +8,7 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import dataclasses import importlib import inspect import math @@ -15,15 +16,16 @@ import time from collections import defaultdict from collections.abc import Callable, Generator, Sequence -from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress +from contextlib import contextmanager, suppress from dataclasses import dataclass, field -from datetime import timedelta +from datetime import date, timedelta from enum import Enum from random import Random from typing import Literal, NoReturn, cast from hypothesis import HealthCheck, Phase, Verbosity, settings as Settings from hypothesis._settings import local_settings +from hypothesis.configuration import storage_directory from hypothesis.database import ExampleDatabase, choices_from_bytes, choices_to_bytes from hypothesis.errors import ( BackendCannotProceed, @@ -68,7 +70,10 @@ from hypothesis.internal.conjecture.shrinker import Shrinker, ShrinkPredicateT, sort_key from hypothesis.internal.escalation import InterestingOrigin from hypothesis.internal.healthcheck import fail_health_check -from hypothesis.internal.observability import Observation, with_observability_callback +from hypothesis.internal.observability import ( + ObservabilityConfig, + Observation, +) from hypothesis.reporting import base_report, report, verbose_report from hypothesis.utils.deprecation import note_deprecation @@ -109,6 +114,9 @@ # deprecate_random_in_strategy). _random = Random() +# used internally to only evict .hypothesis/observed files once per process. +_have_evicted_observations: bool = False + def shortlex(s): return (len(s), s) @@ -921,15 +929,8 @@ def debug_data(self, data: ConjectureData | ConjectureResult) -> None: f"{newline_tab + data.output if data.output else ''}" ) - def observe_for_provider(self) -> AbstractContextManager: - def on_observation(observation: Observation) -> None: - assert observation.type == "test_case" - # because lifetime == "test_function" - assert isinstance(self.provider, PrimitiveProvider) - # only fire if we actually used that provider to generate this observation - if not self._switch_to_hypothesis_provider: - self.provider.on_observation(observation) - + @contextmanager + def observe_for_provider(self): if ( self.settings.backend != "hypothesis" # only for lifetime = "test_function" providers (guaranteed @@ -938,10 +939,46 @@ def on_observation(observation: Observation) -> None: # and the provider opted-in to observations and self.provider.add_observability_callback ): - return with_observability_callback(on_observation) - return nullcontext() + + def callback(observation: Observation) -> None: + assert observation.type == "test_case" + # because lifetime == "test_function" + assert isinstance(self.provider, PrimitiveProvider) + # only fire if we actually used that provider to generate this observation + if not self._switch_to_hypothesis_provider: + self.provider.on_observation(observation) + + old_value = self.settings.observability + self.settings._observability = ( + ObservabilityConfig(callbacks=[callback]) + if old_value is None + else dataclasses.replace( + old_value, callbacks=old_value.callbacks + (callback,) + ) + ) + try: + yield + finally: + self.settings._observability = old_value + else: + yield + + def _maybe_evict_observations(self) -> None: + global _have_evicted_observations + + # Remove files more than a week old, to cap the size on disk. + # Do so no more than once per process, to avoid disk overhead. + if not _have_evicted_observations: + max_age = (date.today() - timedelta(days=8)).isoformat() + for f in storage_directory("observed", intent_to_write=False).glob( + "*.jsonl" + ): # pragma: no cover + if f.stem < max_age: + f.unlink(missing_ok=True) + _have_evicted_observations = True def run(self) -> None: + self._maybe_evict_observations() with local_settings(self.settings), self.observe_for_provider(): try: self._run() diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index f1feae0f5f..a5d6a3c5ef 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -383,7 +383,7 @@ class PrimitiveProvider(abc.ABC): avoid_realization: ClassVar[bool] = False #: If ``True``, |PrimitiveProvider.on_observation| will be added as a - #: callback via |add_observability_callback|, enabling observability during + #: an observability callback, enabling observability during # the lifetime of this provider. If ``False``, |PrimitiveProvider.on_observation| #: will never be called by Hypothesis. #: @@ -583,7 +583,7 @@ def on_observation(self, observation: TestCaseObservation) -> None: # noqa: B02 """ Called at the end of each test case which uses this provider, with the same ``observation["type"] == "test_case"`` observation that is passed to - other callbacks added via |add_observability_callback|. This method is not + other callbacks. This method is not called with ``observation["type"] in {"info", "alert", "error"}`` observations. diff --git a/hypothesis-python/src/hypothesis/internal/observability.py b/hypothesis-python/src/hypothesis/internal/observability.py index ad674f2265..0fd6fdbb4d 100644 --- a/hypothesis-python/src/hypothesis/internal/observability.py +++ b/hypothesis-python/src/hypothesis/internal/observability.py @@ -16,13 +16,10 @@ import math import os import sys -import threading import time -import warnings -from collections.abc import Callable, Generator -from contextlib import contextmanager -from dataclasses import dataclass -from datetime import date, timedelta +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import date from functools import lru_cache from threading import Lock from typing import ( @@ -36,7 +33,7 @@ ) from hypothesis.configuration import storage_directory -from hypothesis.errors import HypothesisWarning +from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture.choice import ( BooleanConstraints, BytesConstraints, @@ -51,22 +48,104 @@ from hypothesis.internal.escalation import InterestingOrigin from hypothesis.internal.floats import float_to_int from hypothesis.internal.intervalsets import IntervalSet +from hypothesis.internal.validation import check_type, try_convert from hypothesis.utils.deprecation import note_deprecation if TYPE_CHECKING: from hypothesis.internal.conjecture.data import ConjectureData, Spans, Status - Observation: TypeAlias = Union["InfoObservation", "TestCaseObservation"] CallbackThreadT: TypeAlias = Callable[[Observation], None] # for all_threads=True, we pass the thread id as well. CallbackAllThreadsT: TypeAlias = Callable[[Observation, int], None] CallbackT: TypeAlias = CallbackThreadT | CallbackAllThreadsT -# thread_id: list[callback] -_callbacks: dict[int | None, list[CallbackThreadT]] = {} -# callbacks where all_threads=True was set -_callbacks_all_threads: list[CallbackAllThreadsT] = [] + +_WROTE_TO = set() +_deliver_to_file_lock = Lock() + + +def _deliver_to_file(observation: Observation) -> None: # pragma: no cover + from hypothesis.strategies._internal.utils import to_jsonable + + kind = "testcases" if observation.type == "test_case" else "info" + fname = storage_directory("observed", f"{date.today().isoformat()}_{kind}.jsonl") + fname.parent.mkdir(exist_ok=True, parents=True) + + observation_bytes = ( + json.dumps(to_jsonable(observation, avoid_realization=False)) + "\n" + ) + # only allow one conccurent file write to avoid write races. This is likely to make + # HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY quite slow under threading. A queue + # would be an improvement, but that requires a background thread, and I + # would prefer to avoid a thread in the single-threaded case. We could + # switch over to a queue if we detect multithreading, but it's tricky to get + # right. + with _deliver_to_file_lock: + _WROTE_TO.add(fname) + with fname.open(mode="a") as f: + f.write(observation_bytes) + + +@dataclass(slots=True, frozen=False) +class ObservabilityConfig: + """ + Options for the |settings.observability| argument to |@settings|. + + Parameters + ---------- + + coverage : bool + Include the ``coverage`` field in test case observations. + choices : bool + Include the ``metadata.choice_nodes`` and ``metadata.choice_spans`` + fields in test case observations. + callbacks : Collection[Callable] + Observability callbacks. Each callback will be called for each observation + this test produces. See :ref:`observability ` for details. + + By default, ``callbacks`` is set to a callback which writes all observations + to a file in ``.hypothesis/observed``. + """ + + coverage: bool = True + choices: bool = False + callbacks: tuple[Callable, ...] = field(default=(_deliver_to_file,)) + + def __post_init__(self): + self.callbacks = try_convert(tuple, self.callbacks, "callbacks") + if not self.callbacks: + raise InvalidArgument( + "Must pass at least one callback to ObservabilityConfig. " + f"Got: {self!r}." + ) + + def __or__(self, other: object) -> "ObservabilityConfig": + if other is None or other is False: + return self + + if other is True: + other = ObservabilityConfig() + + if isinstance(other, ObservabilityConfig): + return ObservabilityConfig( + coverage=self.coverage or other.coverage, + choices=self.choices or other.choices, + callbacks=( + self.callbacks + + tuple(f for f in other.callbacks if f not in self.callbacks) + ), + ) + return NotImplemented + + __ror__ = __or__ + + def union( + self, other: "ObservabilityConfig | bool | None" + ) -> "ObservabilityConfig": + """Returns the union of this ObservabilityConfig with another.""" + check_type((ObservabilityConfig, bool, type(None)), other, "other") + return self | other @dataclass(slots=True, frozen=False) @@ -256,137 +335,19 @@ class TestCaseObservation(BaseObservation): metadata: ObservationMetadata -def add_observability_callback(f: CallbackT, /, *, all_threads: bool = False) -> None: - """ - Adds ``f`` as a callback for :ref:`observability `. ``f`` - should accept one argument, which is an observation. Whenever Hypothesis - produces a new observation, it calls each callback with that observation. - - If Hypothesis tests are being run from multiple threads, callbacks are tracked - per-thread. In other words, ``add_observability_callback(f)`` only adds ``f`` - as an observability callback for observations produced on that thread. - - If ``all_threads=True`` is passed, ``f`` will instead be registered as a - callback for all threads. This means it will be called for observations - generated by all threads, not just the thread which registered ``f`` as a - callback. In this case, ``f`` will be passed two arguments: the first is the - observation, and the second is the integer thread id from - :func:`python:threading.get_ident` where that observation was generated. - - We recommend against registering ``f`` as a callback for both ``all_threads=True`` - and the default ``all_threads=False``, due to unclear semantics with - |remove_observability_callback|. - """ - if all_threads: - _callbacks_all_threads.append(cast(CallbackAllThreadsT, f)) - return - - thread_id = threading.get_ident() - if thread_id not in _callbacks: - _callbacks[thread_id] = [] - - _callbacks[thread_id].append(cast(CallbackThreadT, f)) - - -def remove_observability_callback(f: CallbackT, /) -> None: - """ - Removes ``f`` from the :ref:`observability ` callbacks. - - If ``f`` is not in the list of observability callbacks, silently do nothing. - - If running under multiple threads, ``f`` will only be removed from the - callbacks for this thread. - """ - if f in _callbacks_all_threads: - _callbacks_all_threads.remove(cast(CallbackAllThreadsT, f)) - - thread_id = threading.get_ident() - if thread_id not in _callbacks: - return - - callbacks = _callbacks[thread_id] - if f in callbacks: - callbacks.remove(cast(CallbackThreadT, f)) - - if not callbacks: - del _callbacks[thread_id] - - def observability_enabled() -> bool: - """ - Returns whether or not Hypothesis considers :ref:`observability ` - to be enabled. Observability is enabled if there is at least one observability - callback present. - - Callers might use this method to determine whether they should compute an - expensive representation that is only used under observability, for instance - by |alternative backends|. - """ - return bool(_callbacks) or bool(_callbacks_all_threads) - - -@contextmanager -def with_observability_callback( - f: Callable[[Observation], None], /, *, all_threads: bool = False -) -> Generator[None, None, None]: - """ - A simple context manager which calls |add_observability_callback| on ``f`` - when it enters and |remove_observability_callback| on ``f`` when it exits. - """ - add_observability_callback(f, all_threads=all_threads) - try: - yield - finally: - remove_observability_callback(f) - - -def deliver_observation(observation: Observation) -> None: - thread_id = threading.get_ident() - - for callback in _callbacks.get(thread_id, []): - callback(observation) - - for callback in _callbacks_all_threads: - callback(observation, thread_id) - - -class _TestcaseCallbacks: - def __bool__(self): - self._note_deprecation() - return bool(_callbacks) - - def _note_deprecation(self): - note_deprecation( - "hypothesis.internal.observability.TESTCASE_CALLBACKS is deprecated. " - "Replace TESTCASE_CALLBACKS.append with add_observability_callback, " - "TESTCASE_CALLBACKS.remove with remove_observability_callback, and " - "bool(TESTCASE_CALLBACKS) with observability_enabled().", - since="2025-08-01", - has_codemod=False, - ) - - def append(self, f): - self._note_deprecation() - add_observability_callback(f) - - def remove(self, f): - self._note_deprecation() - remove_observability_callback(f) - - -#: .. warning:: -#: -#: Deprecated in favor of |add_observability_callback|, -#: |remove_observability_callback|, and |observability_enabled|. -#: -#: |TESTCASE_CALLBACKS| remains a thin compatibility -#: shim which forwards ``.append``, ``.remove``, and ``bool()`` to those -#: three methods. It is not an attempt to be fully compatible with the previous -#: ``TESTCASE_CALLBACKS = []``, so iteration or other usages will not work -#: anymore. Please update to using the new methods instead. -#: -#: |TESTCASE_CALLBACKS| will eventually be removed. -TESTCASE_CALLBACKS = _TestcaseCallbacks() + from hypothesis._settings import settings + + note_deprecation( + "observability_enabled() is deprecated. To determine whether observability " + "is enabled, we instead recommend checking " + "`settings().observability is not None`. See the settings.observability " + "docs for details.", + since="RELEASEDAY", + stacklevel=1, + has_codemod=False, + ) + return settings().observability is not None def make_testcase( @@ -408,6 +369,8 @@ def make_testcase( # added to calculated metadata. If keys overlap, the value from this `metadata` # is used metadata: dict[str, Any] | None = None, + # observability settings from settings.observability + observability: ObservabilityConfig | None, ) -> TestCaseObservation: from hypothesis.core import reproduction_decorator from hypothesis.internal.conjecture.data import Status @@ -452,7 +415,7 @@ def make_testcase( }, **data.events, }, - coverage=coverage, + coverage=coverage if observability and observability.coverage else None, timing=timing, metadata=ObservationMetadata( **{ @@ -465,8 +428,12 @@ def make_testcase( "data_status": data.status, "phase": phase, "interesting_origin": data.interesting_origin, - "choice_nodes": data.nodes if OBSERVABILITY_CHOICES else None, - "choice_spans": data.spans if OBSERVABILITY_CHOICES else None, + "choice_nodes": ( + data.nodes if observability and observability.choices else None + ), + "choice_spans": ( + data.spans if observability and observability.choices else None + ), **_system_metadata(), # unpack last so it takes precedence for duplicate keys **(metadata or {}), @@ -477,34 +444,6 @@ def make_testcase( ) -_WROTE_TO = set() -_deliver_to_file_lock = Lock() - - -def _deliver_to_file( - observation: Observation, thread_id: int -) -> None: # pragma: no cover - from hypothesis.strategies._internal.utils import to_jsonable - - kind = "testcases" if observation.type == "test_case" else "info" - fname = storage_directory("observed", f"{date.today().isoformat()}_{kind}.jsonl") - fname.parent.mkdir(exist_ok=True, parents=True) - - observation_bytes = ( - json.dumps(to_jsonable(observation, avoid_realization=False)) + "\n" - ) - # only allow one conccurent file write to avoid write races. This is likely to make - # HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY quite slow under threading. A queue - # would be an improvement, but that requires a background thread, and I - # would prefer to avoid a thread in the single-threaded case. We could - # switch over to a queue if we detect multithreading, but it's tricky to get - # right. - with _deliver_to_file_lock: - _WROTE_TO.add(fname) - with fname.open(mode="a") as f: - f.write(observation_bytes) - - _imported_at = time.time() @@ -517,47 +456,43 @@ def _system_metadata() -> dict[str, Any]: } -#: If ``False``, do not collect coverage information when observability is enabled. -#: -#: This is exposed both for performance (as coverage collection can be slow on -#: Python 3.11 and earlier) and size (if you do not use coverage information, -#: you may not want to store it in-memory). -OBSERVABILITY_COLLECT_COVERAGE = ( - "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_NOCOVER" not in os.environ -) -#: If ``True``, include the ``metadata.choice_nodes`` and ``metadata.spans`` keys -#: in test case observations. -#: -#: ``False`` by default. ``metadata.choice_nodes`` and ``metadata.spans`` can be -#: a substantial amount of data, and so must be opted-in to, even when -#: observability is enabled. -#: -#: .. warning:: -#: -#: EXPERIMENTAL AND UNSTABLE. We are actively working towards a better -#: interface for this as of June 2025, and this attribute may disappear or -#: be renamed without notice. -#: -OBSERVABILITY_CHOICES = "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICES" in os.environ - -if OBSERVABILITY_COLLECT_COVERAGE is False and ( - sys.version_info[:2] >= (3, 12) -): # pragma: no cover - warnings.warn( - "Coverage data collection should be quite fast in Python 3.12 or later " - "so there should be no need to turn coverage reporting off.", - HypothesisWarning, - stacklevel=2, +envvar_observability: ObservabilityConfig | None = None + +# supported for backwards compat. These two were always marked experimental, so +# they can be removed whenever. 6 months would be more than generous. +if ( + envvar_value := os.environ.get("HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY") +) is not None: # pragma: no cover + note_deprecation( + "the HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY environment variable is " + "deprecated. Use either @settings(observability=...), or the " + "HYPOTHESIS_OBSERVABILITY environment variable instead. " + "See the observability docs for details.", + since="RELEASEDAY", + has_codemod=False, + ) + envvar_observability = ObservabilityConfig() + +if ( + envvar_value := os.environ.get("HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICES") +) is not None: # pragma: no cover + note_deprecation( + "the HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICES environment variable is " + "deprecated. Use @settings(observability=ObservabilityConfig(choices=True)) " + "instead. See the settings.observability docs for details.", + since="RELEASEDAY", + has_codemod=False, ) + envvar_observability = ObservabilityConfig(choices=True) if ( - "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY" in os.environ - or OBSERVABILITY_COLLECT_COVERAGE is False -): # pragma: no cover - add_observability_callback(_deliver_to_file, all_threads=True) - - # Remove files more than a week old, to cap the size on disk - max_age = (date.today() - timedelta(days=8)).isoformat() - for f in storage_directory("observed", intent_to_write=False).glob("*.jsonl"): - if f.stem < max_age: # pragma: no branch - f.unlink(missing_ok=True) + envvar_value := os.environ.get("HYPOTHESIS_OBSERVABILITY") +) is not None: # pragma: no cover + allowed = {"true", "false", "1", "0"} + if envvar_value.lower() not in allowed: + raise ValueError( + f"Got {envvar_value!r} for the HYPOTHESIS_OBSERVABILITY environment " + f"variable, but expected one of {allowed!r}" + ) + if envvar_value in {"true", "1"}: + envvar_observability = ObservabilityConfig() diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py index 2103fdcb2c..1a56a916de 100644 --- a/hypothesis-python/src/hypothesis/stateful.py +++ b/hypothesis-python/src/hypothesis/stateful.py @@ -41,7 +41,6 @@ from hypothesis.internal.conjecture.junkdrawer import gc_cumulative_time from hypothesis.internal.conjecture.utils import calc_label_from_name from hypothesis.internal.healthcheck import fail_health_check -from hypothesis.internal.observability import observability_enabled from hypothesis.internal.reflection import ( function_digest, get_pretty_function_description, @@ -132,7 +131,7 @@ def run_state_machine(data): def output(s): if print_steps: report(s) - if observability_enabled(): + if Settings().observability is not None: cd._stateful_repr_parts.append(s) try: @@ -187,7 +186,7 @@ def output(s): # _add_results_to_targets, to avoid printing arguments which are also # a return value using the variable name they are assigned to. # See https://github.com/HypothesisWorks/hypothesis/issues/2341 - if print_steps or observability_enabled(): + if print_steps or Settings().observability is not None: data_to_print = { k: machine._pretty_print(v) for k, v in data.items() } @@ -224,7 +223,7 @@ def output(s): HealthCheck.return_value, ) finally: - if print_steps or observability_enabled(): + if print_steps or Settings().observability is not None: # 'result' is only used if the step has target bundles. # If it does, and the result is a 'MultipleResult', # then 'print_step' prints a multi-variable assignment. @@ -471,7 +470,7 @@ def check_invariants(self, settings, output, runtimes): if ( current_build_context().is_final or settings.verbosity >= Verbosity.debug - or observability_enabled() + or Settings().observability is not None ): output(f"state.{name}()") start = perf_counter() diff --git a/hypothesis-python/tests/common/utils.py b/hypothesis-python/tests/common/utils.py index d84c1946d8..261ff7e95d 100644 --- a/hypothesis-python/tests/common/utils.py +++ b/hypothesis-python/tests/common/utils.py @@ -22,13 +22,7 @@ from hypothesis import Phase, settings from hypothesis.errors import HypothesisDeprecationWarning -from hypothesis.internal import observability from hypothesis.internal.floats import next_down -from hypothesis.internal.observability import ( - Observation, - add_observability_callback, - remove_observability_callback, -) from hypothesis.internal.reflection import get_pretty_function_description, proxies from hypothesis.reporting import default, with_reporter from hypothesis.strategies._internal.core import from_type, register_type_strategy @@ -223,22 +217,6 @@ def raises_warning(expected_warning, match=None): yield r -@contextlib.contextmanager -def capture_observations(*, choices=None): - ls: list[Observation] = [] - add_observability_callback(ls.append) - if choices is not None: - old_choices = observability.OBSERVABILITY_CHOICES - observability.OBSERVABILITY_CHOICES = choices - - try: - yield ls - finally: - remove_observability_callback(ls.append) - if choices is not None: - observability.OBSERVABILITY_CHOICES = old_choices - - # Specifies whether we can represent subnormal floating point numbers. # IEE-754 requires subnormal support, but it's often disabled anyway by unsafe # compiler options like `-ffast-math`. On most hardware that's even a global diff --git a/hypothesis-python/tests/conjecture/test_provider.py b/hypothesis-python/tests/conjecture/test_provider.py index 75f2bbec36..b6eb994d3c 100644 --- a/hypothesis-python/tests/conjecture/test_provider.py +++ b/hypothesis-python/tests/conjecture/test_provider.py @@ -21,6 +21,7 @@ from hypothesis import ( HealthCheck, + ObservabilityConfig, Verbosity, assume, errors, @@ -51,11 +52,10 @@ ) from hypothesis.internal.floats import SIGNALING_NAN, clamp from hypothesis.internal.intervalsets import IntervalSet -from hypothesis.internal.observability import Observation, _callbacks +from hypothesis.internal.observability import Observation, _deliver_to_file from tests.common.debug import minimal from tests.common.utils import ( - capture_observations, capture_out, checks_deprecated_behaviour, ) @@ -378,7 +378,7 @@ def test_function(n): def test_flaky_with_backend(): - with temp_register_backend("trivial", TrivialProvider), capture_observations(): + with temp_register_backend("trivial", TrivialProvider): calls = 0 @@ -482,15 +482,19 @@ def test_function(data): def test_realization_with_observability(): + + observations = [] with temp_register_backend("realize", RealizeProvider): @given(st.data()) - @settings(backend="realize") + @settings( + backend="realize", + observability=ObservabilityConfig(callbacks=[observations.append]), + ) def test_function(data): data.draw(st.integers()) - with capture_observations() as observations: - test_function() + test_function() test_cases = [tc for tc in observations if tc.type == "test_case"] assert {tc.representation for tc in test_cases} == { @@ -520,25 +524,30 @@ def realize(self, value, *, for_failure=False): def test_custom_observations_from_backend(): + observations = [] + with temp_register_backend("observable", ObservableProvider): @given(st.booleans()) - @settings(backend="observable", database=None) + @settings( + backend="observable", + database=None, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) def test_function(_): pass - with capture_observations() as ls: - test_function() + test_function() - assert len(ls) >= 3 - cases = [t.metadata.backend for t in ls if t.type == "test_case"] + assert len(observations) >= 3 + cases = [t.metadata.backend for t in observations if t.type == "test_case"] assert {"msg_key": "some message", "data_key": [1, "2", {}]} in cases - assert "" in repr(ls) + assert "" in repr(observations) infos = [ {k: v for k, v in dataclasses.asdict(t).items() if k in ("title", "content")} - for t in ls + for t in observations if t.type != "test_case" ] assert {"title": "Trivial alert", "content": "message here"} in infos @@ -756,6 +765,8 @@ def on_observation(self, observation: Observation) -> None: @temp_register_backend("observation", ObservationProvider) def test_on_observation_alternates(): + # there are no assertions in this test, because the assertions are inside of + # ObservationProvider @given(st.integers()) @settings(backend="observation") def f(n): @@ -764,6 +775,20 @@ def f(n): f() +@temp_register_backend("observation", ObservationProvider) +def test_backend_adds_observability_callback(): + @given(st.integers()) + @settings(backend="observation") + def f(n): + callbacks = settings().observability.callbacks + # it's hard to reach in and identify the exact local function which *should* + # be in here, but it should not be the standard _deliver_to_file callback. + assert len(callbacks) == 1 + assert _deliver_to_file not in callbacks + + f() + + @temp_register_backend("observation", ObservationProvider) def test_on_observation_alternates_on_failure(): @given(st.integers()) @@ -783,7 +808,7 @@ def test_on_observation_no_override(): @given(st.integers()) @settings(backend="observation") def f(n): - assert _callbacks == {} + assert settings().observability is None f() diff --git a/hypothesis-python/tests/cover/test_direct_strategies.py b/hypothesis-python/tests/cover/test_direct_strategies.py index 02b020e08a..6454c5f90b 100644 --- a/hypothesis-python/tests/cover/test_direct_strategies.py +++ b/hypothesis-python/tests/cover/test_direct_strategies.py @@ -667,7 +667,8 @@ def test_it(a, b): assert a == b with warnings.catch_warnings(): - warnings.simplefilter("error", HypothesisWarning) + # append so we respect temporary ci warning filters + warnings.simplefilter("error", HypothesisWarning, append=True) test_it() diff --git a/hypothesis-python/tests/cover/test_observability.py b/hypothesis-python/tests/cover/test_observability.py index fb31ab81e7..c2388b6f97 100644 --- a/hypothesis-python/tests/cover/test_observability.py +++ b/hypothesis-python/tests/cover/test_observability.py @@ -9,19 +9,17 @@ # obtain one at https://mozilla.org/MPL/2.0/. import base64 -import contextlib import json import math import textwrap import threading -import warnings from collections import defaultdict from contextlib import nullcontext import pytest -import hypothesis.internal.observability from hypothesis import ( + ObservabilityConfig, assume, event, example, @@ -40,15 +38,9 @@ from hypothesis.internal.floats import SIGNALING_NAN, float_to_int, int_to_float from hypothesis.internal.intervalsets import IntervalSet from hypothesis.internal.observability import ( - TESTCASE_CALLBACKS, - InfoObservation, - TestCaseObservation, - add_observability_callback, choices_to_json, nodes_to_json, observability_enabled, - remove_observability_callback, - with_observability_callback, ) from hypothesis.stateful import ( RuleBasedStateMachine, @@ -60,7 +52,6 @@ from tests.common.utils import ( Why, - capture_observations, checks_deprecated_behaviour, run_concurrently, skipif_threading, @@ -69,35 +60,40 @@ from tests.conjecture.common import choices, integer_constr, nodes -@seed("deterministic so we don't miss some combination of features") -@example(l=[1], a=0, x=4, data=None) -# explicitly set max_examples=100 to override our lower example limit for coverage tests. -@settings(database=InMemoryExampleDatabase(), deadline=None, max_examples=100) -@given(st.lists(st.integers()), st.integers(), st.integers(), st.data()) -def do_it_all(l, a, x, data): - event(f"{x%2=}") - target(x % 5, label="x%5") - assume(a % 9) - assume(len(l) > 0) - if data: - data.draw(st.text("abcdef", min_size=a % 3), label="interactive") - 1 / ((x or 1) % 7) - - @xfail_on_crosshair(Why.other, strict=False) # flakey BackendCannotProceed ?? @skipif_threading # captures observations from other threads def test_observability(): - with capture_observations() as ls: - with pytest.raises(ZeroDivisionError): - do_it_all() - with pytest.raises(ZeroDivisionError): - do_it_all() - - infos = [t for t in ls if t.type == "info"] + observations = [] + + @seed("deterministic so we don't miss some combination of features") + @example(l=[1], a=0, x=4, data=None) + # explicitly set max_examples=100 to override our lower example limit for coverage tests. + @settings( + database=InMemoryExampleDatabase(), + deadline=None, + max_examples=100, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) + @given(st.lists(st.integers()), st.integers(), st.integers(), st.data()) + def do_it_all(l, a, x, data): + event(f"{x%2=}") + target(x % 5, label="x%5") + assume(a % 9) + assume(len(l) > 0) + if data: + data.draw(st.text("abcdef", min_size=a % 3), label="interactive") + 1 / ((x or 1) % 7) + + with pytest.raises(ZeroDivisionError): + do_it_all() + with pytest.raises(ZeroDivisionError): + do_it_all() + + infos = [t for t in observations if t.type == "info"] assert len(infos) == 2 assert {t.title for t in infos} == {"Hypothesis Statistics"} - testcases = [t for t in ls if t.type == "test_case"] + testcases = [t for t in observations if t.type == "test_case"] assert len(testcases) > 50 assert {t.property for t in testcases} == {do_it_all.__name__} assert len({t.run_start for t in testcases}) == 2 @@ -112,12 +108,14 @@ def test_observability(): @xfail_on_crosshair(Why.other) def test_capture_unnamed_arguments(): + observations = [] + @given(st.integers(), st.floats(), st.data()) + @settings(observability=ObservabilityConfig(callbacks=[observations.append])) def f(v1, v2, data): data.draw(st.booleans()) - with capture_observations() as observations: - f() + f() test_cases = [tc for tc in observations if tc.type == "test_case"] for test_case in test_cases: @@ -133,13 +131,18 @@ def f(v1, v2, data): PYPY or IN_COVERAGE_TESTS, reason="coverage requires sys.settrace pre-3.12" ) def test_failure_includes_explain_phase_comments(): + observations = [] + @given(st.integers(), st.integers()) - @settings(database=None) + @settings( + database=None, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) def test_fails(x, y): if x: raise AssertionError - with capture_observations() as observations, pytest.raises(AssertionError): + with pytest.raises(AssertionError): test_fails() test_cases = [tc for tc in observations if tc.type == "test_case"] @@ -160,15 +163,20 @@ def test_fails(x, y): def test_failure_includes_notes(): + observations = [] + @given(st.data()) - @settings(database=None) + @settings( + database=None, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) def test_fails_with_note(data): note("not included 1") data.draw(st.booleans()) note("not included 2") raise AssertionError - with capture_observations() as observations, pytest.raises(AssertionError): + with pytest.raises(AssertionError): test_fails_with_note() expected = textwrap.dedent( @@ -184,15 +192,17 @@ def test_fails_with_note(data): def test_normal_representation_includes_draws(): + observations = [] + @given(st.data()) + @settings(observability=ObservabilityConfig(callbacks=[observations.append])) def f(data): b1 = data.draw(st.booleans()) note("not included") b2 = data.draw(st.booleans(), label="second") assume(b1 and b2) - with capture_observations() as observations: - f() + f() crosshair = settings.get_current_profile_name() == "crosshair" expected = textwrap.dedent( @@ -216,12 +226,17 @@ def f(data): @xfail_on_crosshair(Why.other) def test_capture_named_arguments(): + observations = [] + @given(named1=st.integers(), named2=st.floats(), data=st.data()) + @settings( + database=None, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) def f(named1, named2, data): data.draw(st.booleans()) - with capture_observations() as observations: - f() + f() test_cases = [tc for tc in observations if tc.type == "test_case"] for test_case in test_cases: @@ -234,14 +249,18 @@ def f(named1, named2, data): def test_assume_has_status_reason(): + observations = [] + @given(st.booleans()) + @settings(observability=ObservabilityConfig(callbacks=[observations.append])) def f(b): assume(b) - with capture_observations() as ls: - f() + f() - gave_ups = [t for t in ls if t.type == "test_case" and t.status == "gave_up"] + gave_ups = [ + t for t in observations if t.type == "test_case" and t.status == "gave_up" + ] for gave_up in gave_ups: assert gave_up.status_reason.startswith("failed to satisfy assume() in f") @@ -250,13 +269,18 @@ def f(b): PYPY or IN_COVERAGE_TESTS, reason="coverage requires sys.settrace pre-3.12" ) def test_minimal_failing_observation(): + observations = [] + @given(st.integers(), st.integers()) - @settings(database=None) + @settings( + database=None, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) def test_fails(x, y): if x: raise AssertionError - with capture_observations() as observations, pytest.raises(AssertionError): + with pytest.raises(AssertionError): test_fails() observation = [tc for tc in observations if tc.type == "test_case"][-1] @@ -293,11 +317,14 @@ def test_fails(x, y): PYPY or IN_COVERAGE_TESTS, reason="coverage requires sys.settrace pre-3.12" ) def test_all_failing_observations_have_reproduction_decorator(): + observations = [] + @given(st.integers()) + @settings(observability=ObservabilityConfig(callbacks=[observations.append])) def test_fails(x): raise AssertionError - with capture_observations() as observations, pytest.raises(AssertionError): + with pytest.raises(AssertionError): test_fails() # all failed test case observations should have reprodution_decorator @@ -309,29 +336,33 @@ def test_fails(x): assert decorator.startswith("@reproduce_failure") -@settings(max_examples=20, stateful_step_count=5) -class UltraSimpleMachine(RuleBasedStateMachine): - value = 0 +@xfail_on_crosshair(Why.other, strict=False) +def test_observability_captures_stateful_reprs(): + observations = [] - @rule() - def inc(self): - self.value += 1 + @settings( + max_examples=20, + stateful_step_count=5, + observability=ObservabilityConfig(callbacks=[observations.append]), + ) + class UltraSimpleMachine(RuleBasedStateMachine): + value = 0 - @rule() - def dec(self): - self.value -= 1 + @rule() + def inc(self): + self.value += 1 - @invariant() - def limits(self): - assert abs(self.value) <= 100 + @rule() + def dec(self): + self.value -= 1 + @invariant() + def limits(self): + assert abs(self.value) <= 100 -@xfail_on_crosshair(Why.other, strict=False) -def test_observability_captures_stateful_reprs(): - with capture_observations() as ls: - run_state_machine_as_test(UltraSimpleMachine) + run_state_machine_as_test(UltraSimpleMachine) - for x in ls: + for x in observations: if x.type != "test_case" or x.status == "gave_up": continue r = x.representation @@ -361,7 +392,10 @@ def test_observability_captures_stateful_reprs(): ], ) def test_fuzz_one_input_status(buffer, expected_status): + observations = [] + @given(st.booleans(), st.booleans()) + @settings(observability=ObservabilityConfig(callbacks=[observations.append])) def test_fails(should_fail, should_fail_assume): if should_fail: raise AssertionError @@ -369,13 +403,12 @@ def test_fails(should_fail, should_fail_assume): assume(False) with ( - capture_observations() as ls, pytest.raises(AssertionError) if expected_status == "failed" else nullcontext(), ): test_fails.hypothesis.fuzz_one_input(buffer) - assert len(ls) == 1 - assert ls[0].status == expected_status - assert ls[0].how_generated == "fuzz_one_input" + assert len(observations) == 1 + assert observations[0].status == expected_status + assert observations[0].how_generated == "fuzz_one_input" def _decode_choice(value): @@ -534,12 +567,16 @@ def test_choice_nodes_to_json_explicit(choice_node, expected): def test_metadata_to_json(): # this is mostly a covering test than testing anything particular about # ObservationMetadata. + observations = [] + @given(st.integers()) + @settings( + observability=ObservabilityConfig(callbacks=[observations.append], choices=True) + ) def f(n): pass - with capture_observations(choices=True) as observations: - f() + f() observations = [obs for obs in observations if obs.type == "test_case"] for observation in observations: @@ -567,209 +604,93 @@ def f(n): assert 0 <= span.end <= len(observation.metadata.choice_nodes) -@contextlib.contextmanager -def restore_callbacks(): - callbacks = hypothesis.internal.observability._callbacks.copy() - callbacks_all = hypothesis.internal.observability._callbacks_all_threads.copy() - try: - yield - finally: - hypothesis.internal.observability._callbacks = callbacks - hypothesis.internal.observability._callbacks_all_threads = callbacks_all - - -@contextlib.contextmanager -def with_collect_coverage(*, value: bool): - original_value = hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE - hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE = value - try: - yield - finally: - hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE = ( - original_value - ) - - -def _callbacks(): - # respect changes from the restore_callbacks context manager by re-accessing - # its namespace, instead of keeping - # `from hypothesis.internal.observability import _callbacks` around - return hypothesis.internal.observability._callbacks - - -@skipif_threading -def test_observability_callbacks(): - def f(observation): - pass - - def g(observation): - pass - - thread_id = threading.get_ident() - - with restore_callbacks(): - assert not observability_enabled() - - add_observability_callback(f) - assert _callbacks() == {thread_id: [f]} - assert observability_enabled() - - add_observability_callback(g) - assert _callbacks() == {thread_id: [f, g]} - assert observability_enabled() - - remove_observability_callback(g) - assert _callbacks() == {thread_id: [f]} - assert observability_enabled() - - remove_observability_callback(g) - assert _callbacks() == {thread_id: [f]} - assert observability_enabled() - - remove_observability_callback(f) - assert _callbacks() == {} - assert not observability_enabled() - +def test_only_receives_callbacks_from_this_thread(): + n_threads = 5 + count_observations = defaultdict(int) -@skipif_threading -def test_observability_callbacks_all_threads(): - thread_id = threading.get_ident() + def callback(observation): + count_observations[threading.get_ident()] += 1 - def f(observation, thread_id): + @given(st.integers()) + @settings( + observability=ObservabilityConfig( + callbacks=[callback], + # Observability tries to record coverage, but we don't currently + # support concurrent coverage collection, and issue a warning instead. + # + # I tried to fix this with: + # + # warnings.filterwarnings( + # "ignore", message=r".*tool id \d+ is already taken by tool scrutineer.*" + # ) + # + # but that had a race condition somehow and sometimes still didn't work?? The + # warnings module is not thread-safe until 3.14, I think. + coverage=False, + ) + ) + def f(n): pass - with restore_callbacks(): - assert not observability_enabled() - - add_observability_callback(f, all_threads=True) - assert hypothesis.internal.observability._callbacks_all_threads == [f] - assert _callbacks() == {} - assert observability_enabled() - - add_observability_callback(f) - assert hypothesis.internal.observability._callbacks_all_threads == [f] - assert _callbacks() == {thread_id: [f]} - assert observability_enabled() - - # remove_observability_callback removes it both from per-thread and - # all_threads. The semantics of duplicated callbacks is weird enough - # that I don't want to commit to anything here, so I'm leaving this as - # somewhat undefined behavior, and recommending that users simply not - # register a callback both normally and for all threads. - remove_observability_callback(f) - assert hypothesis.internal.observability._callbacks_all_threads == [] - assert _callbacks() == {} - assert not observability_enabled() - + run_concurrently(f, n=n_threads) -@checks_deprecated_behaviour -def test_testcase_callbacks_deprecation_bool(): - bool(TESTCASE_CALLBACKS) + assert len(count_observations) == n_threads + # one observation per example, plus one info observation for the overall run + assert set(count_observations.values()) == {settings().max_examples + 1} @checks_deprecated_behaviour -def test_testcase_callbacks_deprecation_append(): - with restore_callbacks(): - TESTCASE_CALLBACKS.append(lambda x: None) - - -@checks_deprecated_behaviour -def test_testcase_callbacks_deprecation_remove(): - with restore_callbacks(): - TESTCASE_CALLBACKS.remove(lambda x: None) - - -def test_testcase_callbacks(): - def f(observation): - pass - - def g(observation): - pass - - thread_id = threading.get_ident() +def test_observability_enabled_is_deprecated(): + observability_enabled() - with restore_callbacks(), warnings.catch_warnings(): - # ignore TESTCASE_CALLBACKS deprecation warnings - warnings.simplefilter("ignore") - assert not bool(TESTCASE_CALLBACKS) - add_observability_callback(f) - assert _callbacks() == {thread_id: [f]} +@pytest.mark.parametrize("other", [None, False]) +def test_config_union_returns_self(other): + config = ObservabilityConfig(callbacks=[lambda _obs: None]) + assert config | other is config + assert other | config is config - assert bool(TESTCASE_CALLBACKS) - add_observability_callback(g) - assert _callbacks() == {thread_id: [f, g]} - assert bool(TESTCASE_CALLBACKS) - remove_observability_callback(g) - assert _callbacks() == {thread_id: [f]} +def test_observability_config_or_with_true_merges_with_default(): + cb = lambda _obs: None + config = ObservabilityConfig(coverage=False, choices=True, callbacks=[cb]) - assert bool(TESTCASE_CALLBACKS) - remove_observability_callback(f) - assert _callbacks() == {} + for result in [config | True, True | config]: + assert result.coverage is True + assert result.choices is True + assert cb in result.callbacks + assert len(result.callbacks) == 2 - assert not bool(TESTCASE_CALLBACKS) +@pytest.mark.parametrize("left", [False, True]) +@pytest.mark.parametrize("right", [False, True]) +def test_config_union(left, right): + cb1 = lambda _obs: None + cb2 = lambda _obs: None + config1 = ObservabilityConfig(coverage=left, choices=left, callbacks=[cb1]) + config2 = ObservabilityConfig(coverage=right, choices=right, callbacks=[cb2]) -def test_only_receives_callbacks_from_this_thread(): - @given(st.integers()) - def g(n): - pass - - def test(): - count_observations = 0 - - def callback(observation): - nonlocal count_observations - count_observations += 1 - - add_observability_callback(callback) - - with warnings.catch_warnings(): - g() - - # one per example, plus one for the overall run - assert count_observations == settings().max_examples + 1 - - with ( - restore_callbacks(), - # Observability tries to record coverage, but we don't currently - # support concurrent coverage collection, and issue a warning instead. - # - # I tried to fix this with: - # - # warnings.filterwarnings( - # "ignore", message=r".*tool id \d+ is already taken by tool scrutineer.*" - # ) - # - # but that had a race condition somehow and sometimes still didn't work?? The - # warnings module is not thread-safe until 3.14, I think. - with_collect_coverage(value=False), - ): - run_concurrently(test, n=5) - - -def test_all_threads_callback(): - n_threads = 5 - - # thread_id: count - calls = defaultdict(int) + for result in [config1 | config2, config1.union(config2)]: + assert result.coverage is (left or right) + assert result.choices is (left or right) + assert result.callbacks == (cb1, cb2) - def global_callback(observation, thread_id): - assert isinstance(observation, (TestCaseObservation, InfoObservation)) - assert isinstance(thread_id, int) - calls[thread_id] += 1 +def test_config_deduplicates_callbacks(): + cb1 = lambda _obs: None + cb2 = lambda _obs: None + cb3 = lambda _obs: None + config1 = ObservabilityConfig(callbacks=[cb1, cb2]) + config2 = ObservabilityConfig(callbacks=[cb2, cb3]) - @given(st.integers()) - def f(n): - pass + assert (config1 | config2).callbacks == (cb1, cb2, cb3) - with ( - with_collect_coverage(value=False), - with_observability_callback(global_callback, all_threads=True), - ): - run_concurrently(f, n=n_threads) - assert len(calls) == n_threads - assert all(count == (settings().max_examples + 1) for count in calls.values()) +def test_config_invalid_union(): + config = ObservabilityConfig(callbacks=[lambda _obs: None]) + with pytest.raises(TypeError): + config | "invalid" + with pytest.raises(TypeError): + "invalid" | config + with pytest.raises(TypeError): + config.union("invalid") diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py index 754d04dd63..71b30d9ab4 100644 --- a/hypothesis-python/tests/cover/test_settings.py +++ b/hypothesis-python/tests/cover/test_settings.py @@ -9,6 +9,7 @@ # obtain one at https://mozilla.org/MPL/2.0/. import datetime +import math import os import subprocess import sys @@ -33,6 +34,10 @@ HypothesisDeprecationWarning, InvalidArgument, ) +from hypothesis.internal.observability import ( + ObservabilityConfig, + _deliver_to_file, +) from hypothesis.stateful import RuleBasedStateMachine, rule from hypothesis.utils.conventions import not_set from hypothesis.utils.deprecation import note_deprecation @@ -463,6 +468,8 @@ def test_derandomise_with_explicit_database_is_invalid(): {"phases": 0}, {"verbosity": -1}, {"verbosity": "nonexistent_verbosity"}, + {"observability": "bad_option"}, + {"observability": 10}, ], ) def test_invalid_settings_are_errors(kwargs): @@ -684,3 +691,35 @@ def test_invalid_integer_phase_raises(): def test_invalid_integer_healthcheck_raises(): with pytest.raises(ValueError): HealthCheck(99) + + +def test_settings_observability(): + s = settings(observability=True) + assert s.observability == ObservabilityConfig() + + s = settings(observability=False) + assert s.observability is None + + s = settings(observability=None) + assert s.observability is None + + s = settings(observability=ObservabilityConfig(coverage=True)) + assert s.observability == ObservabilityConfig(coverage=True) + + s = settings(observability=ObservabilityConfig(choices=True)) + assert s.observability == ObservabilityConfig(choices=True) + + s = settings(observability=ObservabilityConfig(coverage=True, choices=True)) + assert s.observability == ObservabilityConfig(coverage=True, choices=True) + + with pytest.raises(InvalidArgument): + _ = ObservabilityConfig(callbacks=math.inf) + + # invalid to create an ObservabilityConfig with no callbacks + with pytest.raises(InvalidArgument): + _ = ObservabilityConfig(callbacks=[]) + + +def test_settings_observability_has_callback_by_default(): + s = settings(observability=True) + assert s.observability.callbacks == (_deliver_to_file,) diff --git a/hypothesis-python/tests/cover/test_validation.py b/hypothesis-python/tests/cover/test_validation.py index 750628966f..432c0e67e1 100644 --- a/hypothesis-python/tests/cover/test_validation.py +++ b/hypothesis-python/tests/cover/test_validation.py @@ -293,7 +293,8 @@ def f(s): # if we reorder, it doesn't warn anymore with warnings.catch_warnings(): - warnings.simplefilter("error") + # append so we respect temporary ci warning filters + warnings.simplefilter("error", append=True) @given(st.text(codec[1:] + codec[:1])) def f(s): diff --git a/hypothesis-python/tests/crosshair/test_crosshair.py b/hypothesis-python/tests/crosshair/test_crosshair.py index 8e8bb6812a..e98aa7b8b9 100644 --- a/hypothesis-python/tests/crosshair/test_crosshair.py +++ b/hypothesis-python/tests/crosshair/test_crosshair.py @@ -12,14 +12,20 @@ import pytest from hypothesis_crosshair_provider.crosshair_provider import CrossHairPrimitiveProvider -from hypothesis import Phase, Verbosity, event, given, settings, strategies as st +from hypothesis import ( + ObservabilityConfig, + Phase, + Verbosity, + event, + given, + settings, + strategies as st, +) from hypothesis.database import InMemoryExampleDatabase from hypothesis.internal.conjecture.providers import COLLECTION_DEFAULT_MAX_SIZE from hypothesis.internal.intervalsets import IntervalSet -from hypothesis.internal.observability import with_observability_callback from hypothesis.vendor.pretty import pretty -from tests.common.utils import capture_observations from tests.conjecture.common import float_constr, integer_constr, string_constr @@ -189,17 +195,23 @@ def f_verbosity(value): called = False @given(strategy) - @settings(backend="crosshair", database=None, max_examples=2) + @settings( + backend="crosshair", + database=None, + max_examples=2, + # pass an arbitrary callback, to enable observability + observability=ObservabilityConfig(callbacks=[lambda observation: None]), + ) def f_observability(value): nonlocal called + assert settings().observability is not None if called: choices["observability"] = len( crosshair.statespace.context_statespace().choices_made ) called = True - with capture_observations(): - f_observability() + f_observability() assert ( choices["normal"] @@ -223,12 +235,15 @@ def callback(observation): saw_myevent = True @given(st.integers()) - @settings(backend="crosshair", max_examples=5) + @settings( + backend="crosshair", + max_examples=5, + observability=ObservabilityConfig(callbacks=[callback]), + ) def test(n): event("myevent", n) - with with_observability_callback(callback): - test() + test() assert saw_myevent diff --git a/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt b/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt index e6e72805ec..f299c00fd2 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt @@ -10,6 +10,19 @@ from collections.abc import Hashable from hypothesis import given, settings, strategies as st +@given( + coverage=st.booleans(), + choices=st.booleans(), + callbacks=st.lists(st.functions()).map(tuple), +) +def test_fuzz_ObservabilityConfig( + coverage: bool, choices: bool, callbacks: tuple +) -> None: + hypothesis.ObservabilityConfig( + coverage=coverage, choices=choices, callbacks=callbacks + ) + + @given(condition=st.from_type(object)) def test_fuzz_assume(condition: object) -> None: hypothesis.assume(condition=condition) @@ -81,6 +94,7 @@ def test_fuzz_seed(seed: collections.abc.Hashable) -> None: deadline=st.just(not_set), print_blob=st.just(not_set), backend=st.just(not_set), + observability=st.just(not_set), ) def test_fuzz_settings( parent: typing.Optional[hypothesis.settings], @@ -95,6 +109,7 @@ def test_fuzz_settings( deadline: typing.Union[int, float, datetime.timedelta, None], print_blob: bool, backend: str, + observability: typing.Union[bool, hypothesis.ObservabilityConfig, None], ) -> None: hypothesis.settings( parent=parent, @@ -109,6 +124,7 @@ def test_fuzz_settings( deadline=deadline, print_blob=print_blob, backend=backend, + observability=observability, ) diff --git a/hypothesis-python/tests/nocover/test_regressions.py b/hypothesis-python/tests/nocover/test_regressions.py index 9544b7d978..ecd409878a 100644 --- a/hypothesis-python/tests/nocover/test_regressions.py +++ b/hypothesis-python/tests/nocover/test_regressions.py @@ -11,8 +11,8 @@ import pytest from hypothesis import given, settings, strategies as st -from hypothesis._settings import note_deprecation from hypothesis.errors import HypothesisDeprecationWarning +from hypothesis.utils.deprecation import note_deprecation from tests.common.utils import Why, xfail_on_crosshair diff --git a/pyproject.toml b/pyproject.toml index 7c387fdce0..99346c4f26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,4 +131,6 @@ filterwarnings = [ # pytest-cov can't see into subprocesses; we'll see <100% covered if this is an issue "ignore:Module hypothesis.* was previously imported, but not measured", "ignore:CrosshairPrimitiveProvider.realize does not have the for_failure parameter", + # remove once crosshair updates to checking settings().observability + "ignore:observability_enabled", ]