From 79346d80ebc2bbb9b74688711467cd96d9bb464a Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 2 Jan 2026 01:37:18 -0500 Subject: [PATCH] rename "example" to "test case" in many places --- README.md | 6 +- hypothesis-python/RELEASE.rst | 7 ++ hypothesis-python/docs/quickstart.rst | 2 +- hypothesis-python/docs/reference/api.rst | 2 +- .../docs/tutorial/adding-notes.rst | 6 +- hypothesis-python/docs/tutorial/flaky.rst | 2 +- .../docs/tutorial/replaying-failures.rst | 6 +- .../src/_hypothesis_pytestplugin.py | 4 +- hypothesis-python/src/hypothesis/__init__.py | 2 +- hypothesis-python/src/hypothesis/_settings.py | 42 +++++------ hypothesis-python/src/hypothesis/control.py | 2 +- hypothesis-python/src/hypothesis/core.py | 32 ++++----- hypothesis-python/src/hypothesis/database.py | 2 +- .../src/hypothesis/extra/_patching.py | 6 +- .../hypothesis/internal/conjecture/engine.py | 10 +-- .../internal/conjecture/providers.py | 4 +- .../internal/conjecture/shrinker.py | 8 +-- .../src/hypothesis/internal/scrutineer.py | 2 +- .../src/hypothesis/statistics.py | 7 +- .../hypothesis/strategies/_internal/core.py | 10 +-- .../src/hypothesis/vendor/pretty.py | 4 +- hypothesis-python/tests/common/utils.py | 10 ++- .../tests/conjecture/test_inquisitor.py | 8 +-- .../tests/conjecture/test_provider.py | 2 +- .../tests/cover/test_custom_reprs.py | 8 +-- .../tests/cover/test_deadline.py | 4 +- .../tests/cover/test_explicit_examples.py | 12 ++-- .../cover/test_falsifying_example_output.py | 2 +- .../tests/cover/test_fuzz_one_input.py | 6 +- .../tests/cover/test_observability.py | 2 +- hypothesis-python/tests/cover/test_pretty.py | 2 +- .../tests/cover/test_reporting.py | 2 +- .../tests/cover/test_slippage.py | 4 +- .../tests/cover/test_stateful.py | 22 +++--- .../tests/cover/test_statistical_events.py | 8 +-- .../tests/cover/test_testdecorators.py | 8 +-- .../tests/cover/test_verbosity.py | 8 +-- .../tests/nocover/test_skipping.py | 6 +- .../tests/nocover/test_targeting.py | 2 +- .../tests/patching/test_patching.py | 2 +- .../tests/pytest/test_capture.py | 2 +- .../tests/pytest/test_reporting.py | 2 +- .../tests/pytest/test_skipping.py | 8 +-- .../content/2016-04-16-anatomy-of-a-test.md | 70 +++++++++---------- .../2016-04-16-encode-decode-invariant.md | 4 +- .../2016-04-19-rule-based-stateful-testing.md | 2 +- ...04-29-testing-performance-optimizations.md | 2 +- ...5-29-testing-optimizers-with-hypothesis.md | 4 +- ...-06-13-testing-configuration-parameters.md | 4 +- ...-06-30-tests-as-complete-specifications.md | 2 +- .../2016-07-04-calculating-the-mean.md | 8 +-- .../content/2016-07-23-what-is-hypothesis.md | 2 +- .../2017-03-09-hypothesis-for-researchers.md | 4 +- .../content/2017-09-14-multi-bug-discovery.md | 4 +- website/content/2018-01-08-smarkets.md | 2 +- 55 files changed, 208 insertions(+), 196 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/README.md b/README.md index b4a2f96fb8..d59d279189 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ def test_matches_builtin(ls): assert sorted(ls) == my_sort(ls) ``` -This randomized testing can catch bugs and edge cases that you didn't think of and wouldn't have found. In addition, when Hypothesis does find a bug, it doesn't just report any failing example — it reports the simplest possible one. This makes property-based tests a powerful tool for debugging, as well as testing. +This randomized testing can catch bugs and edge cases that you didn't think of and wouldn't have found. In addition, when Hypothesis does find a bug, it doesn't just report any failing test case — it reports the simplest possible one. This makes property-based tests a powerful tool for debugging, as well as testing. For instance, @@ -30,10 +30,10 @@ def my_sort(ls): return sorted(set(ls)) ``` -fails with the simplest possible failing example: +fails with the simplest possible failing test case: ``` -Falsifying example: test_matches_builtin(ls=[0, 0]) +Failing test case: test_matches_builtin(ls=[0, 0]) ``` ### Installation diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..3aa5283c57 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,7 @@ +RELEASE_TYPE: minor + +Hypothesis has historically referred to the single execution of a test function as an "example". However, we have in recent years tended to call a single execution a "test case" instead, which we think is a more precise and less overloaded term. + +This release updates user-facing documentation, log messages, and error messages to use the "test case" terminology. + +This is not a breaking change, as we have intentionally not changed any code APIs. For example, |settings.max_examples| has not been changed. diff --git a/hypothesis-python/docs/quickstart.rst b/hypothesis-python/docs/quickstart.rst index 43219d413b..09ee385a83 100644 --- a/hypothesis-python/docs/quickstart.rst +++ b/hypothesis-python/docs/quickstart.rst @@ -70,7 +70,7 @@ This test will clearly fail, which can be confirmed by running ``pytest example. def test_integers(n): > assert n < 50 E assert 50 < 50 - E Falsifying example: test_integers( + E Failing test case: test_integers( E n=50, E ) diff --git a/hypothesis-python/docs/reference/api.rst b/hypothesis-python/docs/reference/api.rst index a382a34ab5..575c0f9fb8 100644 --- a/hypothesis-python/docs/reference/api.rst +++ b/hypothesis-python/docs/reference/api.rst @@ -121,7 +121,7 @@ For instance, in the latter case, you would see output like: - during generate phase (0.09 seconds): - Typical runtimes: < 1ms, ~ 59% in data generation - - 100 passing examples, 0 failing examples, 32 invalid examples + - 100 passing test cases, 0 failing test cases, 32 invalid test cases - Events: * 54.55%, Retried draw from integers().filter(lambda x: x % 2 == 0) to satisfy filter * 31.06%, i mod 3 = 2 diff --git a/hypothesis-python/docs/tutorial/adding-notes.rst b/hypothesis-python/docs/tutorial/adding-notes.rst index 004815455f..b326c7214d 100644 --- a/hypothesis-python/docs/tutorial/adding-notes.rst +++ b/hypothesis-python/docs/tutorial/adding-notes.rst @@ -5,7 +5,7 @@ When a test fails, Hypothesis will normally print output that looks like this: .. code:: - Falsifying example: test_a_thing(x=1, y="foo") + Failing test case: test_a_thing(x=1, y="foo") Sometimes you want to add some additional information to a failure, such as the output of some intermediate step in your test. The |note| function lets you do this: @@ -24,11 +24,11 @@ Sometimes you want to add some additional information to a failure, such as the ... except AssertionError: ... print("ls != ls2") ... - Falsifying example: test_shuffle_is_noop(ls=[0, 1], r=RandomWithSeed(1)) + Failing test case: test_shuffle_is_noop(ls=[0, 1], r=RandomWithSeed(1)) Shuffle: [1, 0] ls != ls2 -|note| is like a print statement that gets attached to the falsifying example reported by Hypothesis. It's also reported by :ref:`observability `, and shown for all examples (if |settings.verbosity| is set to |Verbosity.verbose| or higher). +|note| is like a print statement that gets attached to the failing test case reported by Hypothesis. It's also reported by :ref:`observability `, and shown for all examples (if |settings.verbosity| is set to |Verbosity.verbose| or higher). .. note:: diff --git a/hypothesis-python/docs/tutorial/flaky.rst b/hypothesis-python/docs/tutorial/flaky.rst index 488478e8a6..a2cdbf805a 100644 --- a/hypothesis-python/docs/tutorial/flaky.rst +++ b/hypothesis-python/docs/tutorial/flaky.rst @@ -61,7 +61,7 @@ As a result, running ``test_fails_flakily()`` will raise |FlakyFailure|. |FlakyF + Exception Group Traceback (most recent call last): | hypothesis.errors.FlakyFailure: Hypothesis test_fails_flakily(n=0) produces unreliable results: Falsified on the first call but did not on a subsequent one (1 sub-exception) - | Falsifying example: test_fails_flakily( + | Failing test case: test_fails_flakily( | n=0, | ) | Failed to reproduce exception. Expected: diff --git a/hypothesis-python/docs/tutorial/replaying-failures.rst b/hypothesis-python/docs/tutorial/replaying-failures.rst index ed986c60fe..0c5eb00b73 100644 --- a/hypothesis-python/docs/tutorial/replaying-failures.rst +++ b/hypothesis-python/docs/tutorial/replaying-failures.rst @@ -67,7 +67,7 @@ Note that unlike examples generated by Hypothesis, examples provided using |@exa def test_something_with_integers(n): assert n < 100 -Hypothesis will print ``Falsifying explicit example: test_something_with_integers(n=131071)``, instead of shrinking to ``n=100``. +Hypothesis will print ``Failing explicit example: test_something_with_integers(n=131071)``, instead of shrinking to ``n=100``. Prefer |@example| over the database for correctness --------------------------------------------------- @@ -93,10 +93,10 @@ If |settings.print_blob| is set to ``True`` (the default in the ``ci`` settings >>> test() ... - Falsifying example: test( + Failing test case: test( f=nan, ) - You can reproduce this example by temporarily adding @reproduce_failure('6.131.23', b'ACh/+AAAAAAAAA==') as a decorator on your test case + You can reproduce this test case by temporarily adding @reproduce_failure('6.131.23', b'ACh/+AAAAAAAAA==') as a decorator on your test function You can add this decorator to your test to reproduce the failure. This can be useful for locally replaying failures found by CI. Note that the binary blob is not stable across Hypothesis versions, so you should not leave this decorator on your tests permanently. Use |@example| with an explicit input instead. diff --git a/hypothesis-python/src/_hypothesis_pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py index 340bf90ebf..199d2c2062 100644 --- a/hypothesis-python/src/_hypothesis_pytestplugin.py +++ b/hypothesis-python/src/_hypothesis_pytestplugin.py @@ -405,7 +405,7 @@ def pytest_terminal_summary(terminalreporter): terminalreporter.write_line(f"observations written to {fname}") if failing_examples: - # This must have been imported already to write the failing examples + # This must have been imported already to write the failing test cases from hypothesis.extra._patching import gc_patches, make_patch, save_patch patch = make_patch(failing_examples) @@ -418,7 +418,7 @@ def pytest_terminal_summary(terminalreporter): if not _WROTE_TO: terminalreporter.section("Hypothesis") terminalreporter.write_line( - f"`git apply {fname}` to add failing examples to your code." + f"`git apply {fname}` to add failing test cases to your code." ) def pytest_collection_modifyitems(items): diff --git a/hypothesis-python/src/hypothesis/__init__.py b/hypothesis-python/src/hypothesis/__init__.py index 0efe43985d..42ba7b390d 100644 --- a/hypothesis-python/src/hypothesis/__init__.py +++ b/hypothesis-python/src/hypothesis/__init__.py @@ -12,7 +12,7 @@ some source of data. It verifies your code against a wide range of input and minimizes any -failing examples it finds. +failing test cases it finds. """ import _hypothesis_globals diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index 13c90373ed..7becc2fe07 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -68,12 +68,12 @@ class Verbosity(Enum): quiet = "quiet" """ - Hypothesis will not print any output, not even the final falsifying example. + Hypothesis will not print any output, not even the final failing test case. """ normal = "normal" """ - Standard verbosity. Hypothesis will print the falsifying example, alongside + Standard verbosity. Hypothesis will print the failing test case, alongside any notes made with |note| (only for the falsfying example). """ @@ -173,7 +173,7 @@ class Phase(Enum): Controls whether Hypothesis attempts to explain test failures. The explain phase has two parts, each of which is best-effort - if Hypothesis - can't find a useful explanation, we'll just print the minimal failing example. + can't find a useful explanation, we'll just print the minimal failing test case. """ @classmethod @@ -851,18 +851,18 @@ def verbosity(self): ... def f(x): ... assert not any(x) ... f() - Trying example: [] - Falsifying example: [-1198601713, -67, 116, -29578] - Shrunk example to [-1198601713] - Shrunk example to [-128] - Shrunk example to [32] - Shrunk example to [1] + Test case: [] + Failing test case: [-1198601713, -67, 116, -29578] + Shrunk test case to [-1198601713] + Shrunk test case to [-128] + Shrunk test case to [32] + Shrunk test case to [1] [1] The four levels are |Verbosity.quiet|, |Verbosity.normal|, |Verbosity.verbose|, and |Verbosity.debug|. |Verbosity.normal| is the default. For |Verbosity.quiet|, Hypothesis will not print anything out, - not even the final falsifying example. |Verbosity.debug| is basically + not even the final failing test case. |Verbosity.debug| is basically |Verbosity.verbose| but a bit more so. You probably don't want it. Verbosity can be passed either as a |Verbosity| enum value, or as the @@ -889,16 +889,16 @@ def phases(self): Hypothesis divides tests into logically distinct phases. - |Phase.explicit|: Running explicit examples from |@example|. - - |Phase.reuse|: Running examples from the database which previously failed. - - |Phase.generate|: Generating new random examples. - - |Phase.target|: Mutating examples for :ref:`targeted property-based + - |Phase.reuse|: Running test cases from the database which previously failed. + - |Phase.generate|: Generating new random test cases. + - |Phase.target|: Mutating test cases for :ref:`targeted property-based testing `. Requires |Phase.generate|. - - |Phase.shrink|: Shrinking failing examples. + - |Phase.shrink|: Shrinking failing test cases. - |Phase.explain|: Attempting to explain why a failure occurred. Requires |Phase.shrink|. The phases argument accepts a collection with any subset of these. E.g. - ``settings(phases=[Phase.generate, Phase.shrink])`` will generate new examples + ``settings(phases=[Phase.generate, Phase.shrink])`` will generate new test cases and shrink them, but will not run explicit examples or reuse previous failures, while ``settings(phases=[Phase.explicit])`` will only run explicit examples from |@example|. @@ -921,10 +921,10 @@ def phases(self): there are no clearly suspicious lines of code, :pep:`we refuse the temptation to guess <20>`. - After shrinking to a minimal failing example, Hypothesis will try to find - parts of the example -- e.g. separate args to |@given| + After shrinking to a minimal failing test case, Hypothesis will try to find + parts of the test case -- e.g. separate args to |@given| -- which can vary freely without changing the result - of that minimal failing example. If the automated experiments run without + of that minimal failing test case. If the automated experiments run without finding a passing variation, we leave a comment in the final report: .. code-block:: python @@ -936,7 +936,7 @@ def phases(self): Just remember that the *lack* of an explanation sometimes just means that Hypothesis couldn't efficiently find one, not that no explanation (or - simpler failing example) exists. + simpler failing test case) exists. """ return self._phases @@ -1011,8 +1011,8 @@ def deadline(self): @property def print_blob(self): """ - If set to ``True``, Hypothesis will print code for failing examples that - can be used with |@reproduce_failure| to reproduce the failing example. + If set to ``True``, Hypothesis will print code for failing test cases that + can be used with |@reproduce_failure| to reproduce the failing test case. The default value is ``False``. If running on CI, the default is ``True`` instead. """ diff --git a/hypothesis-python/src/hypothesis/control.py b/hypothesis-python/src/hypothesis/control.py index 11f8223231..80bf9c4667 100644 --- a/hypothesis-python/src/hypothesis/control.py +++ b/hypothesis-python/src/hypothesis/control.py @@ -237,7 +237,7 @@ def should_note(): def note(value: object) -> None: - """Report this value for the minimal failing example.""" + """Report this value for the minimal failing test case.""" if should_note(): if not isinstance(value, str): value = pretty(value) diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 66fb9fcba1..d84d7d57e4 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -647,7 +647,7 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s with contextlib.suppress(StopTest): empty_data.conclude_test(Status.INVALID) except BaseException as err: - # In order to support reporting of multiple failing examples, we yield + # In order to support reporting of multiple failing test cases, we yield # each of the (report text, error) pairs we find back to the top-level # runner. This also ensures that user-facing stack traces have as few # frames of Hypothesis internals as possible. @@ -682,9 +682,9 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s break finally: if fragments_reported: - assert fragments_reported[0].startswith("Falsifying example") + assert fragments_reported[0].startswith("Failing test case") fragments_reported[0] = fragments_reported[0].replace( - "Falsifying example", "Falsifying explicit example", 1 + "Failing test case", "Failing explicit example", 1 ) empty_data.freeze() @@ -700,7 +700,7 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s deliver_observation(tc) if fragments_reported: - verbose_report(fragments_reported[0].replace("Falsifying", "Trying", 1)) + verbose_report(fragments_reported[0].replace("Failing", "Trying", 1)) for f in fragments_reported[1:]: verbose_report(f) @@ -1055,9 +1055,9 @@ def run(data: ConjectureData) -> None: if print_example or current_verbosity() >= Verbosity.verbose: printer = RepresentationPrinter(context=context) if print_example: - printer.text("Falsifying example:") + printer.text("Failing test case:") else: - printer.text("Trying example:") + printer.text("Test case:") if self.print_given_args: printer.text(" ") @@ -1533,12 +1533,12 @@ def run_engine(self): finally: ran_example.freeze() if observability_enabled(): - # log our observability line for the final failing example + # log our observability line for the final failing test case tc = make_testcase( run_start=self._start_timestamp, property=self.test_identifier, data=ran_example, - how_generated="minimal failing example", + how_generated="minimal failing test case", representation=self._string_repr, arguments=ran_example._observability_args, timing=self._timing_features, @@ -1550,12 +1550,12 @@ def run_engine(self): deliver_observation(tc) # Whether or not replay actually raised the exception again, we want - # to print the reproduce_failure decorator for the failing example. + # to print the reproduce_failure decorator for the failing test case. if self.settings.print_blob: fragments.append( - "\nYou can reproduce this example by temporarily adding " + "\nYou can reproduce this test case by temporarily adding " f"{reproduction_decorator(falsifying_example.choices)} " - "as a decorator on your test case" + "as a decorator on your test function" ) _raise_to_user( @@ -1611,7 +1611,7 @@ def _raise_to_user( errors_to_report, settings, target_lines, trailer="", *, unsound_backend=None ): """Helper function for attaching notes and grouping multiple errors.""" - failing_prefix = "Falsifying example: " + failing_prefix = "Failing test case: " ls = [] for error in errors_to_report: for note in error.fragments: @@ -1648,14 +1648,14 @@ def _raise_to_user( def fake_subTest(self, msg=None, **__): """Monkeypatch for `unittest.TestCase.subTest` during `@given`. - If we don't patch this out, each failing example is reported as a + If we don't patch this out, each failing test case is reported as a separate failing test by the unittest test runner, which is obviously incorrect. We therefore replace it for the duration with this version. """ warnings.warn( - "subTest per-example reporting interacts badly with Hypothesis " - "trying hundreds of examples, so we disable it for the duration of " + "subTest per-test-case reporting interacts badly with Hypothesis " + "trying hundreds of test cases, so we disable it for the duration of " "any test that uses `@given`.", HypothesisWarning, stacklevel=2, @@ -2179,7 +2179,7 @@ def wrapped_test(*arguments, **kwargs): and getattr(wrapped_test, "hypothesis_explicit_examples", ()) ) SKIP_BECAUSE_NO_EXAMPLES = unittest.SkipTest( - "Hypothesis has been told to run no examples for this test." + "Hypothesis has been told to run no test cases for this test." ) if not ( Phase.reuse in settings.phases or Phase.generate in settings.phases diff --git a/hypothesis-python/src/hypothesis/database.py b/hypothesis-python/src/hypothesis/database.py index 9106a2d219..334ea7f505 100644 --- a/hypothesis-python/src/hypothesis/database.py +++ b/hypothesis-python/src/hypothesis/database.py @@ -765,7 +765,7 @@ class GitHubArtifactDatabase(ExampleDatabase): You can use this for sharing example databases between CI runs and developers, allowing the latter to get read-only access to the former. This is particularly useful for continuous fuzzing (i.e. with `HypoFuzz `_), - where the CI system can help find new failing examples through fuzzing, + where the CI system can help find new failing test cases through fuzzing, and developers can reproduce them locally without any manual effort. .. note:: diff --git a/hypothesis-python/src/hypothesis/extra/_patching.py b/hypothesis-python/src/hypothesis/extra/_patching.py index 3dba3f7b5c..a39afb6b54 100644 --- a/hypothesis-python/src/hypothesis/extra/_patching.py +++ b/hypothesis-python/src/hypothesis/extra/_patching.py @@ -13,8 +13,8 @@ Requires `hypothesis[codemods,ghostwriter]` installed, i.e. black and libcst. -This module is used by Hypothesis' builtin pytest plugin for failing examples -discovered during testing, and by HypoFuzz for _covering_ examples discovered +This module is used by Hypothesis' builtin pytest plugin for failing test cases +discovered during testing, and by HypoFuzz for _covering_ test cases discovered during fuzzing. """ @@ -98,7 +98,7 @@ def __init__( [m.Arg(m.SimpleString() & value_in_strip_via)], ) - # Codemod the failing examples to Call nodes usable as decorators + # Codemod the failing test cases to Call nodes usable as decorators self.fn_examples = { k: tuple( d diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 21ea97c7c5..c51bf2f46f 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -84,7 +84,7 @@ # If the shrinking phase takes more than five minutes, abort it early and print # a warning. Many CI systems will kill a build after around ten minutes with # no output, and appearing to hang isn't great for interactive use either - -# showing partially-shrunk examples is better than quitting with no examples! +# showing partially-shrunk test cases is better than quitting with no test cases! # (but make it monkeypatchable, for the rare users who need to keep on shrinking) #: The maximum total time in seconds that the shrinker will try to shrink a failure @@ -161,7 +161,7 @@ class ExitReason(Enum): "settings.max_examples={s.max_examples}, " "but < 10% of examples satisfied assumptions" ) - max_shrinks = f"shrunk example {MAX_SHRINKS} times" + max_shrinks = f"shrunk test case {MAX_SHRINKS} times" finished = "nothing left to do" flaky = "test was flaky" very_slow_shrinking = "shrinking was very slow" @@ -690,7 +690,7 @@ def test_function(self, data: ConjectureData) -> None: # See https://github.com/HypothesisWorks/hypothesis/issues/2340 report( "WARNING: Hypothesis has spent more than five minutes working to shrink" - " a failing example, and stopped because it is making very slow" + " a failing test case, and stopped because it is making very slow" " progress. When you re-run your tests, shrinking will resume and may" " take this long before aborting again.\nPLEASE REPORT THIS if you can" " provide a reproducing example, so that we can improve shrinking" @@ -1099,12 +1099,12 @@ def generate_new_examples(self) -> None: if Phase.generate not in self.settings.phases: return if self.interesting_examples: - # The example database has failing examples from a previous run, + # The example database has failing test cases from a previous run, # so we'd rather report that they're still failing ASAP than take # the time to look for additional failures. return - self.debug("Generating new examples") + self.debug("Generating new test cases") assert self.should_generate_more() self._switch_to_hypothesis_provider = True diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index f1feae0f5f..1debc0a2cc 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -537,10 +537,10 @@ def realize(self, value: T, *, for_failure: bool = False) -> T: The returned value should be non-symbolic. If you cannot provide a value, raise |BackendCannotProceed| with a value of ``"discard_test_case"``. - If ``for_failure`` is ``True``, the value is associated with a failing example. + If ``for_failure`` is ``True``, the value is associated with a failing test case. In this case, the backend should spend substantially more effort when attempting to realize the value, since it is important to avoid discarding - failing examples. Backends may still raise |BackendCannotProceed| when + failing test cases. Backends may still raise |BackendCannotProceed| when ``for_failure`` is ``True``, if realization is truly impossible or if realization takes significantly longer than expected (say, 5 minutes). """ diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index 8b5af7f613..1a462573c6 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -471,18 +471,18 @@ def explain(self) -> None: # Before we start running experiments, let's check for known inputs which would # make them redundant. The shrinking process means that we've already tried many - # variations on the minimal example, so this can save a lot of time. + # variations on the minimal test case, so this can save a lot of time. seen_passing_seq = self.engine.passing_choice_sequences( prefix=self.nodes[: min(self.shrink_target.arg_slices)[0]] ) - # Now that we've shrunk to a minimal failing example, it's time to try + # Now that we've shrunk to a minimal failing test case, it's time to try # varying each part that we've noted will go in the final report. Consider # slices in largest-first order for start, end in sorted( self.shrink_target.arg_slices, key=lambda x: (-(x[1] - x[0]), x) ): - # Check for any previous examples that match the prefix and suffix, + # Check for any previous test cases that match the prefix and suffix, # so we can skip if we found a passing example while shrinking. if any( startswith(seen, nodes[:start]) and endswith(seen, nodes[end:]) @@ -1594,7 +1594,7 @@ def minimize_individual_choices(self, chooser): x = data.draw(integers()) assert x < 10 - then in our shrunk example, x = 10 rather than say 97. + then in our shrunk test case, x = 10 rather than say 97. If we are unsuccessful at minimizing a choice of interest we then check if that's because it's changing the size of the test case and, diff --git a/hypothesis-python/src/hypothesis/internal/scrutineer.py b/hypothesis-python/src/hypothesis/internal/scrutineer.py index a4665d1375..7393b59ecb 100644 --- a/hypothesis-python/src/hypothesis/internal/scrutineer.py +++ b/hypothesis-python/src/hypothesis/internal/scrutineer.py @@ -240,7 +240,7 @@ def get_explaining_locations(traces): EXPLANATION_STUB = ( "Explanation:", - " These lines were always and only run by failing examples:", + " These lines were always and only run by failing test cases:", ) diff --git a/hypothesis-python/src/hypothesis/statistics.py b/hypothesis-python/src/hypothesis/statistics.py index bdb29670c9..f73001aec6 100644 --- a/hypothesis-python/src/hypothesis/statistics.py +++ b/hypothesis-python/src/hypothesis/statistics.py @@ -31,7 +31,7 @@ def note_statistics(stats_dict: "StatisticsDict") -> None: def describe_targets(best_targets: dict[str, float]) -> list[str]: """Return a list of lines describing the results of `target`, if any.""" # These lines are included in the general statistics description below, - # but also printed immediately below failing examples to alleviate the + # but also printed immediately below failing test cases to alleviate the # "threshold problem" where shrinking can make severe bug look trivial. # See https://github.com/HypothesisWorks/hypothesis/issues/2180 if not best_targets: @@ -92,8 +92,9 @@ def describe_statistics(stats_dict: "StatisticsDict") -> str: lines.append( f" - during {phase} phase ({d['duration-seconds']:.2f} seconds):\n" f" - Typical runtimes: {runtime_ms}, of which {drawtime_ms} in data generation\n" - f" - {statuses['valid']} passing examples, {statuses['interesting']} " - f"failing examples, {statuses['invalid'] + statuses['overrun']} invalid examples" + f" - {statuses['valid']} passing test cases, {statuses['interesting']} " + f"failing test cases, {statuses['invalid'] + statuses['overrun']} " + "invalid test cases" ) # If we've found new distinct failures in this phase, report them distinct_failures = d["distinct-failures"] - prev_failures diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py index bd3be431a2..f6927c7ee0 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py @@ -968,7 +968,7 @@ def randoms( are of a special HypothesisRandom subclass. - If ``note_method_calls`` is set to ``True``, Hypothesis will print the - randomly drawn values in any falsifying test case. This can be helpful + randomly drawn values in any failing test case. This can be helpful for debugging the behaviour of randomized algorithms. - If ``use_true_random`` is set to ``True`` then values will be drawn from their usual distribution, otherwise they will actually be Hypothesis @@ -2360,18 +2360,18 @@ def test_values(data): n2 = data.draw(st.integers(min_value=n1)) assert n1 + 1 <= n2 - If the test fails, each draw will be printed with the falsifying example. + If the test fails, each draw will be printed with the failing test case. e.g. the above is wrong (it has a boundary condition error), so will print: .. code-block:: pycon - Falsifying example: test_values(data=data(...)) + Failing test case: test_values(data=data(...)) Draw 1: 0 Draw 2: 0 Optionally, you can provide a label to identify values generated by each call to ``data.draw()``. These labels can be used to identify values in the - output of a falsifying example. + output of a failing test case. For instance: @@ -2387,7 +2387,7 @@ def test_draw_sequentially(data): .. code-block:: pycon - Falsifying example: test_draw_sequentially(data=data(...)) + Failing test case: test_draw_sequentially(data=data(...)) Draw 1 (First number): 0 Draw 2 (Second number): 0 diff --git a/hypothesis-python/src/hypothesis/vendor/pretty.py b/hypothesis-python/src/hypothesis/vendor/pretty.py index af790bbab2..c64df22b92 100644 --- a/hypothesis-python/src/hypothesis/vendor/pretty.py +++ b/hypothesis-python/src/hypothesis/vendor/pretty.py @@ -142,7 +142,7 @@ def __init__( We use the context to represent objects constructed by strategies by showing *how* they were constructed, and add annotations showing which parts of the - minimal failing example can vary without changing the test result. + minimal failing test case can vary without changing the test result. """ self.broken: bool = False self.output: TextIOBase = StringIO() if output is None else output @@ -173,7 +173,7 @@ def __init__( self.deferred_pprinters.update(_deferred_type_pprinters) # for which-parts-matter, we track a mapping from the (start_idx, end_idx) - # of slices into the minimal failing example; this is per-interesting_origin + # of slices into the minimal failing test case; this is per-interesting_origin # but we report each separately so that's someone else's problem here. # Invocations of self.repr_call() can report the slice for each argument, # which will then be used to look up the relevant comment if any. diff --git a/hypothesis-python/tests/common/utils.py b/hypothesis-python/tests/common/utils.py index d84c1946d8..efb243a26a 100644 --- a/hypothesis-python/tests/common/utils.py +++ b/hypothesis-python/tests/common/utils.py @@ -174,8 +174,12 @@ def assert_output_contains_failure(output, test, **kwargs): assert f"{k}={v!r}" in output, (f"{k}={v!r}", output) -def assert_falsifying_output( - test, example_type="Falsifying", expected_exception=AssertionError, **kwargs +def assert_failing_output( + test, + example_type="Failing test case", + *, + expected_exception=AssertionError, + **kwargs, ): with capture_out() as out: if expected_exception is None: @@ -189,7 +193,7 @@ def assert_falsifying_output( msg = str(exc_info.value) + "\n" + notes output = out.getvalue() + msg - assert f"{example_type} example:" in output + assert f"{example_type}:" in output, (example_type, output) assert_output_contains_failure(output, test, **kwargs) diff --git a/hypothesis-python/tests/conjecture/test_inquisitor.py b/hypothesis-python/tests/conjecture/test_inquisitor.py index 8aae5b5146..f226bcde13 100644 --- a/hypothesis-python/tests/conjecture/test_inquisitor.py +++ b/hypothesis-python/tests/conjecture/test_inquisitor.py @@ -40,7 +40,7 @@ def _new(): @fails_with_output( """ -Falsifying example: test_inquisitor_comments_basic_fail_if_either( +Failing test case: test_inquisitor_comments_basic_fail_if_either( # The test always failed when commented parts were varied together. a=False, # or any other generated value b=True, @@ -58,7 +58,7 @@ def test_inquisitor_comments_basic_fail_if_either(a, b, c, d, e): @fails_with_output( """ -Falsifying example: test_inquisitor_comments_basic_fail_if_not_all( +Failing test case: test_inquisitor_comments_basic_fail_if_not_all( # The test sometimes passed when commented parts were varied together. a='', # or any other generated value b='', # or any other generated value @@ -75,7 +75,7 @@ def test_inquisitor_comments_basic_fail_if_not_all(a, b, c): @fails_with_output( """ -Falsifying example: test_inquisitor_no_together_comment_if_single_argument( +Failing test case: test_inquisitor_no_together_comment_if_single_argument( a='', b='', # or any other generated value ) @@ -97,7 +97,7 @@ def ints_with_forced_draw(draw): @fails_with_output( """ -Falsifying example: test_inquisitor_doesnt_break_on_varying_forced_nodes( +Failing test case: test_inquisitor_doesnt_break_on_varying_forced_nodes( n1=100, n2=0, # or any other generated value ) diff --git a/hypothesis-python/tests/conjecture/test_provider.py b/hypothesis-python/tests/conjecture/test_provider.py index ade75e9ca6..4281d93860 100644 --- a/hypothesis-python/tests/conjecture/test_provider.py +++ b/hypothesis-python/tests/conjecture/test_provider.py @@ -464,7 +464,7 @@ def test_function(f): with capture_out() as out: test_function() - assert "Trying example: test_function(\n f=,\n)" in out.getvalue() + assert "Test case: test_function(\n f=,\n)" in out.getvalue() @pytest.mark.parametrize("verbosity", [Verbosity.verbose, Verbosity.debug]) diff --git a/hypothesis-python/tests/cover/test_custom_reprs.py b/hypothesis-python/tests/cover/test_custom_reprs.py index 54db2cfdd4..4847f5a0c7 100644 --- a/hypothesis-python/tests/cover/test_custom_reprs.py +++ b/hypothesis-python/tests/cover/test_custom_reprs.py @@ -88,7 +88,7 @@ def inner(foo, bar, baz): with pytest.raises(AssertionError) as err: inner() expected = """ -Falsifying example: inner( +Failing test case: inner( foo=Foo(x=1), bar=Bar(x=-1), baz=Foo(None), @@ -107,7 +107,7 @@ def inner(data): with pytest.raises(AssertionError) as err: inner() expected = """ -Falsifying example: inner( +Failing test case: inner( data=data(...), ) Draw 1: Bar(10) @@ -133,7 +133,7 @@ def inner(a, b): with pytest.raises(AssertionError) as err: inner() expected_re = r""" -Falsifying example: inner\( +Failing test case: inner\( a=<.*Foo object at 0x[0-9A-Fa-f]+>, b=<.*Foo object at 0x[0-9A-Fa-f]+>, \) @@ -156,7 +156,7 @@ def inner(a, b): with pytest.raises(AssertionError) as err: inner() expected = f""" -Falsifying example: inner( +Failing test case: inner( a=some_foo({aas!r}), b=Bar( some_foo( diff --git a/hypothesis-python/tests/cover/test_deadline.py b/hypothesis-python/tests/cover/test_deadline.py index 4c762f5f59..e17c29c2e8 100644 --- a/hypothesis-python/tests/cover/test_deadline.py +++ b/hypothesis-python/tests/cover/test_deadline.py @@ -17,7 +17,7 @@ from hypothesis import given, settings, strategies as st from hypothesis.errors import DeadlineExceeded, FlakyFailure, InvalidArgument -from tests.common.utils import assert_falsifying_output, fails_with +from tests.common.utils import assert_failing_output, fails_with pytestmark = pytest.mark.skipif( settings.get_current_profile_name() == "threading", @@ -78,7 +78,7 @@ def slow_if_large(i): if i >= 1000: time.sleep(1) - assert_falsifying_output( + assert_failing_output( slow_if_large, expected_exception=DeadlineExceeded, i=1000, diff --git a/hypothesis-python/tests/cover/test_explicit_examples.py b/hypothesis-python/tests/cover/test_explicit_examples.py index 2069b4bb7e..392516e321 100644 --- a/hypothesis-python/tests/cover/test_explicit_examples.py +++ b/hypothesis-python/tests/cover/test_explicit_examples.py @@ -28,7 +28,7 @@ from hypothesis.strategies import floats, integers, text from tests.common.utils import ( - assert_falsifying_output, + assert_failing_output, capture_out, fails_with, skipif_threading, @@ -152,7 +152,7 @@ def test_positive(x): ): test_positive() out = out.getvalue() - assert "Falsifying example: test_positive(1)" not in out + assert "Failing test case: test_positive(1)" not in out def test_prints_output_for_explicit_examples(): @@ -161,7 +161,7 @@ def test_prints_output_for_explicit_examples(): def test_positive(x): assert x > 0 - assert_falsifying_output(test_positive, "Falsifying explicit", x=-1) + assert_failing_output(test_positive, "Failing explicit example", x=-1) def test_prints_verbose_output_for_explicit_examples(): @@ -171,10 +171,10 @@ def test_prints_verbose_output_for_explicit_examples(): def test_always_passes(x): pass - assert_falsifying_output( + assert_failing_output( test_always_passes, expected_exception=None, - example_type="Trying explicit", + example_type="Trying explicit example", x="NOT AN INTEGER", ) @@ -186,7 +186,7 @@ def test_mutation(x): x.append(1) assert not x - assert_falsifying_output(test_mutation, "Falsifying explicit", x=[]) + assert_failing_output(test_mutation, "Failing explicit example", x=[]) def test_examples_are_tried_in_order(): diff --git a/hypothesis-python/tests/cover/test_falsifying_example_output.py b/hypothesis-python/tests/cover/test_falsifying_example_output.py index 95929a08a3..a1f41d0cf5 100644 --- a/hypothesis-python/tests/cover/test_falsifying_example_output.py +++ b/hypothesis-python/tests/cover/test_falsifying_example_output.py @@ -13,7 +13,7 @@ from hypothesis import Phase, example, given, settings, strategies as st OUTPUT_WITH_BREAK = """ -Falsifying explicit example: test( +Failing explicit example: test( x={0!r}, y={0!r}, ) diff --git a/hypothesis-python/tests/cover/test_fuzz_one_input.py b/hypothesis-python/tests/cover/test_fuzz_one_input.py index d0e62d332f..029f8fafb1 100644 --- a/hypothesis-python/tests/cover/test_fuzz_one_input.py +++ b/hypothesis-python/tests/cover/test_fuzz_one_input.py @@ -33,7 +33,7 @@ def test_fuzz_one_input(buffer_type): # This is a standard `@given` test, which we can also use as a fuzz target. # Note that we specify the DB so we can make more precise assertions, - # and tighten the phases so we can be sure the failing examples come from fuzzing. + # and tighten the phases so we can be sure the failing test cases come from fuzzing. @given(st.text()) @settings(database=db, phases=[Phase.reuse, Phase.shrink]) def test(s): @@ -47,7 +47,7 @@ def test(s): assert len(seen) == 0 # If we run a lot of random bytestrings through fuzz_one_input, we'll eventually - # find a failing example. + # find a failing test case. with pytest.raises(AssertionError): for _ in range(1000): buf = randbytes(1000) @@ -57,7 +57,7 @@ def test(s): # fuzz_one_input returns False for invalid bytestrings, due to e.g. assume(False) assert len(seen) <= len(seeds) - # `db` contains exactly one failing example, which is either the most + # `db` contains exactly one failing test case, which is either the most # recent seed that we tried or the pruned-and-canonicalised form of it. (saved_examples,) = db.data.values() assert len(saved_examples) == 1 diff --git a/hypothesis-python/tests/cover/test_observability.py b/hypothesis-python/tests/cover/test_observability.py index fb31ab81e7..74ce1cd4b5 100644 --- a/hypothesis-python/tests/cover/test_observability.py +++ b/hypothesis-python/tests/cover/test_observability.py @@ -281,7 +281,7 @@ def test_fails(x, y): } assert observation.coverage is None assert observation.features == {} - assert observation.how_generated == "minimal failing example" + assert observation.how_generated == "minimal failing test case" assert "AssertionError" in observation.metadata.traceback assert "test_fails" in observation.metadata.traceback assert observation.metadata.reproduction_decorator.startswith("@reproduce_failure") diff --git a/hypothesis-python/tests/cover/test_pretty.py b/hypothesis-python/tests/cover/test_pretty.py index 445be051ba..961386c712 100644 --- a/hypothesis-python/tests/cover/test_pretty.py +++ b/hypothesis-python/tests/cover/test_pretty.py @@ -722,7 +722,7 @@ def __repr__(self): @given(st.data()) def test_pprint_with_call_or_repr_as_call(data): - # mapped pprint repr only triggers for failing examples - which makes an + # mapped pprint repr only triggers for failing test cases - which makes an # end to end test given hypothesis difficult. fake our way around it. current_build_context().is_final = True diff --git a/hypothesis-python/tests/cover/test_reporting.py b/hypothesis-python/tests/cover/test_reporting.py index 5a3eeaa3f3..0042504aa4 100644 --- a/hypothesis-python/tests/cover/test_reporting.py +++ b/hypothesis-python/tests/cover/test_reporting.py @@ -28,7 +28,7 @@ def test_int(x): with pytest.raises(AssertionError) as err: test_int() - assert "Falsifying example" in "\n".join(err.value.__notes__) + assert "Failing test case" in "\n".join(err.value.__notes__) def test_does_not_print_debug_in_verbose(): diff --git a/hypothesis-python/tests/cover/test_slippage.py b/hypothesis-python/tests/cover/test_slippage.py index 0f4d1cad07..cf92800d6e 100644 --- a/hypothesis-python/tests/cover/test_slippage.py +++ b/hypothesis-python/tests/cover/test_slippage.py @@ -348,12 +348,12 @@ def test(x): seen.add(x) assert x - # On the first run, we look for up to ten examples: + # On the first run, we look for up to ten test cases: with pytest.raises(AssertionError): test() assert 1 < len(seen) <= MIN_TEST_CALLS - # With failing examples in the database, we stop at one. + # With failing test cases in the database, we stop at one. seen.clear() with pytest.raises(AssertionError): test() diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py index d5f75aee84..e84cd79ac0 100644 --- a/hypothesis-python/tests/cover/test_stateful.py +++ b/hypothesis-python/tests/cover/test_stateful.py @@ -295,7 +295,7 @@ def fail_fast(self): run_state_machine_as_test(ProducesMultiple) # This is tightly coupled to the output format of the step printing. - # The first line is "Falsifying Example:..." the second is creating + # The first line is "Failing test case:..." the second is creating # the state machine, the third is calling the "initialize" method. assignment_line = err.value.__notes__[2] # 'populate_bundle()' returns 2 values, so should be @@ -352,7 +352,7 @@ def fail_fast(self): run_state_machine_as_test(ProducesNoVariables) # This is tightly coupled to the output format of the step printing. - # The first line is "Falsifying Example:..." the second is creating + # The first line is "Failing test case:..." the second is creating # the state machine, the third is calling the "initialize" method. assignment_line = err.value.__notes__[2] # 'populate_bundle()' returns 0 values, so there should be no @@ -734,7 +734,7 @@ def rule_1(self): assert ( result == """ -Falsifying example: +Failing test case: state = BadInvariant() state.initialize_1() state.invariant_1() @@ -777,7 +777,7 @@ def rule_1(self): run_state_machine_as_test(BadRuleWithGoodInvariants) expected = """ -Falsifying example: +Failing test case: state = BadRuleWithGoodInvariants() state.invariant_1() state.initialize_1() @@ -948,7 +948,7 @@ def fail_fast(self, param): assert ( result == """ -Falsifying example: +Failing test case: state = WithInitializeBundleRules() a_0 = state.initialize_a(dep='dep') state.fail_fast(param=a_0) @@ -1079,7 +1079,7 @@ def fail_eventually(self): assert ( result == """ -Falsifying example: +Failing test case: state = StateMachine() state.initialize() state.fail_eventually() @@ -1102,7 +1102,7 @@ def oops(self): assert ( "\n".join(err.value.__notes__).strip() == """ -Falsifying example: +Failing test case: state = RaisesProblem() state.oops() state.teardown()""".strip() @@ -1319,7 +1319,7 @@ def fail_fast(self, param): assert ( result == """ -Falsifying example: +Failing test case: state = Machine() a_0, a_1, a_2 = state.initialize() state.fail_fast(param=a_2) @@ -1371,7 +1371,7 @@ def fail_fast(self): assert ( result == f""" -Falsifying example: +Failing test case: state = Machine() {repr_} state.fail_fast() @@ -1408,7 +1408,7 @@ def fail_fast(self, a1, a2, a3, b1, b2, b3): assert ( result == """ -Falsifying example: +Failing test case: state = Machine() a_0, a_1, a_2 = state.initialize() b_0, b_1, b_2 = a_0, a_1, a_2 @@ -1449,7 +1449,7 @@ def fail_fast(self, a1, a2, a3, a4, a5, a6, b1, b2, b3): assert ( result == """ -Falsifying example: +Failing test case: state = Machine() a_0, a_1, a_2 = state.initialize() b_0, b_1, b_2 = a_0, a_1, a_2 diff --git a/hypothesis-python/tests/cover/test_statistical_events.py b/hypothesis-python/tests/cover/test_statistical_events.py index 3c274eaeaa..e2cc9fdbf9 100644 --- a/hypothesis-python/tests/cover/test_statistical_events.py +++ b/hypothesis-python/tests/cover/test_statistical_events.py @@ -197,7 +197,7 @@ def test(n): assert n < 10 stats = call_for_statistics(test) - assert "shrunk example" in stats["stopped-because"] + assert "shrunk test case" in stats["stopped-because"] def test_stateful_states_are_deduped(): @@ -248,12 +248,12 @@ def test_statistics_for_threshold_problem(): def threshold(error): target(error, label="error") assert error <= 10 - target(0.0, label="never in failing example") + target(0.0, label="never in failing test case") stats = call_for_statistics(threshold) assert " - Highest target scores:" in describe_statistics(stats) - assert "never in failing example" in describe_statistics(stats) - # Check that we report far-from-threshold failing examples + assert "never in failing test case" in describe_statistics(stats) + # Check that we report far-from-threshold failing test cases assert stats["targets"]["error"] > 10 diff --git a/hypothesis-python/tests/cover/test_testdecorators.py b/hypothesis-python/tests/cover/test_testdecorators.py index 33a5509c22..94750a79d5 100644 --- a/hypothesis-python/tests/cover/test_testdecorators.py +++ b/hypothesis-python/tests/cover/test_testdecorators.py @@ -43,7 +43,7 @@ from tests.common.utils import ( Why, - assert_falsifying_output, + assert_failing_output, capture_out, fails, fails_with, @@ -230,7 +230,7 @@ def test_ints_are_sorted(balthazar, evans): assume(evans >= 0) assert balthazar <= evans - assert_falsifying_output(test_ints_are_sorted, balthazar=1, evans=0) + assert_failing_output(test_ints_are_sorted, balthazar=1, evans=0) def test_does_not_print_on_success(): @@ -405,7 +405,7 @@ def test_mixed_text(x): assert set(x).issubset(set("abcdefg")) -@xfail_on_crosshair(Why.other, strict=False) # runs ~five failing examples +@xfail_on_crosshair(Why.other, strict=False) # runs ~five failing test cases def test_when_set_to_no_simplifies_runs_failing_example_twice(): failing = [] @@ -426,7 +426,7 @@ def foo(x): foo() assert len(failing) == 2 assert len(set(failing)) == 1 - assert "Falsifying example" in "\n".join(err.value.__notes__) + assert "Failing test case" in "\n".join(err.value.__notes__) assert "Lo" in err.value.__notes__ diff --git a/hypothesis-python/tests/cover/test_verbosity.py b/hypothesis-python/tests/cover/test_verbosity.py index 17ea8ee2cb..fdbda259bd 100644 --- a/hypothesis-python/tests/cover/test_verbosity.py +++ b/hypothesis-python/tests/cover/test_verbosity.py @@ -34,7 +34,7 @@ def test_works(x): pass test_works() - assert "Trying example" in o.getvalue() + assert "Test case:" in o.getvalue() def test_does_not_log_in_quiet_mode(): @@ -59,7 +59,7 @@ def test_includes_progress_in_verbose_mode(): ) out = o.getvalue() assert out - assert "Trying example: " in out + assert "Test case: " in out @xfail_on_crosshair(Why.symbolic_outside_context, strict=False) @@ -83,7 +83,7 @@ def not_first(x): foo() - assert "Trying example" in o.getvalue() + assert "Test case:" in o.getvalue() def test_includes_intermediate_results_in_verbose_mode(): @@ -102,7 +102,7 @@ def test_foo(x): test_foo() lines = o.getvalue().splitlines() - assert len([l for l in lines if "example" in l]) > 2 + assert len([l for l in lines if "Test case:" in l]) > 2 assert [l for l in lines if "AssertionError" in l] diff --git a/hypothesis-python/tests/nocover/test_skipping.py b/hypothesis-python/tests/nocover/test_skipping.py index 33fd2e68c8..b4ab2e0c94 100644 --- a/hypothesis-python/tests/nocover/test_skipping.py +++ b/hypothesis-python/tests/nocover/test_skipping.py @@ -22,10 +22,10 @@ @pytest.mark.parametrize("skip_exception", skip_exceptions_to_reraise()) -def test_no_falsifying_example_if_unittest_skip(skip_exception): +def test_no_failing_test_case_if_unittest_skip(skip_exception): """If a ``SkipTest`` exception is raised during a test, Hypothesis should not continue running the test and shrink process, nor should it print - anything about falsifying examples.""" + anything about failing test cases.""" class DemoTest(unittest.TestCase): @given(xs=integers()) @@ -39,7 +39,7 @@ def test_to_be_skipped(self, xs): suite = unittest.defaultTestLoader.loadTestsFromTestCase(DemoTest) unittest.TextTestRunner().run(suite) - assert "Falsifying example" not in o.getvalue() + assert "Failing test case" not in o.getvalue() def test_skip_exceptions_save_database_entries(): diff --git a/hypothesis-python/tests/nocover/test_targeting.py b/hypothesis-python/tests/nocover/test_targeting.py index 1165f95ee2..73cbe331df 100644 --- a/hypothesis-python/tests/nocover/test_targeting.py +++ b/hypothesis-python/tests/nocover/test_targeting.py @@ -33,7 +33,7 @@ def test_reports_target_results(testdir, multiple): script = testdir.makepyfile(TESTSUITE.format("" if multiple else "# ")) result = testdir.runpytest(script, "--tb=native", "-rN") out = "\n".join(result.stdout.lines) - assert "Falsifying example" in out + assert "Failing test case" in out assert "x=101" in out, out assert out.count("Highest target score") == 1 assert result.ret != 0 diff --git a/hypothesis-python/tests/patching/test_patching.py b/hypothesis-python/tests/patching/test_patching.py index c727ff9adf..738797c914 100644 --- a/hypothesis-python/tests/patching/test_patching.py +++ b/hypothesis-python/tests/patching/test_patching.py @@ -248,7 +248,7 @@ def test_pytest_reports_patch_file_location(pytester): result.assert_outcomes(failed=1) fname_pat = r"\.hypothesis/patches/\d{4}-\d\d-\d\d--[0-9a-f]{8}.patch" - pattern = f"`git apply ({fname_pat})` to add failing examples to your code\\." + pattern = f"`git apply ({fname_pat})` to add failing test cases to your code\\." print(f"{pattern=}") print(f"result.stdout=\n{indent(str(result.stdout), ' ')}") fname = re.search(pattern, str(result.stdout)).group(1) diff --git a/hypothesis-python/tests/pytest/test_capture.py b/hypothesis-python/tests/pytest/test_capture.py index ac750c81ec..28507a243c 100644 --- a/hypothesis-python/tests/pytest/test_capture.py +++ b/hypothesis-python/tests/pytest/test_capture.py @@ -33,7 +33,7 @@ def test_output_without_capture(testdir, capture, expected): result = testdir.runpytest(script, "--verbose", "--capture", capture) out = "\n".join(result.stdout.lines) assert "test_should_be_verbose" in out - assert ("Trying example" in out) == expected + assert ("Test case:" in out) == expected assert result.ret == 0 diff --git a/hypothesis-python/tests/pytest/test_reporting.py b/hypothesis-python/tests/pytest/test_reporting.py index a7cb778752..e3578fa8d0 100644 --- a/hypothesis-python/tests/pytest/test_reporting.py +++ b/hypothesis-python/tests/pytest/test_reporting.py @@ -33,7 +33,7 @@ def test_runs_reporting_hook(testdir): out = "\n".join(result.stdout.lines) assert "test_this_one_is_ok" in out assert "Captured stdout call" not in out - assert "Falsifying example" in out + assert "Failing test case" in out assert result.ret != 0 diff --git a/hypothesis-python/tests/pytest/test_skipping.py b/hypothesis-python/tests/pytest/test_skipping.py index af92d3aae4..50775e98df 100644 --- a/hypothesis-python/tests/pytest/test_skipping.py +++ b/hypothesis-python/tests/pytest/test_skipping.py @@ -29,22 +29,22 @@ def test_to_be_skipped(xs): """ -def test_no_falsifying_example_if_pytest_skip(testdir): +def test_no_failing_test_case_if_pytest_skip(testdir): """If ``pytest.skip() is called during a test, Hypothesis should not continue running the test and shrink process, nor should it print anything - about falsifying examples.""" + about failing test cases.""" script = testdir.makepyfile(PYTEST_TESTSUITE) result = testdir.runpytest( script, "--verbose", "--strict-markers", "-m", "hypothesis" ) out = "\n".join(result.stdout.lines) - assert "Falsifying example" not in out + assert "Failing test case" not in out def test_issue_3453_regression(testdir): """If ``pytest.skip() is called during a test, Hypothesis should not continue running the test and shrink process, nor should it print anything - about falsifying examples.""" + about failing test cases.""" script = testdir.makepyfile( """ from hypothesis import example, given, strategies as st diff --git a/website/content/2016-04-16-anatomy-of-a-test.md b/website/content/2016-04-16-anatomy-of-a-test.md index 6d81147bae..a0093732af 100644 --- a/website/content/2016-04-16-anatomy-of-a-test.md +++ b/website/content/2016-04-16-anatomy-of-a-test.md @@ -41,7 +41,7 @@ E assert (0.0 + nan) == (nan + 0.0) test_floats.py:7: AssertionError -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` The test fails, because [nan](https://en.wikipedia.org/wiki/NaN) is a valid floating @@ -78,40 +78,40 @@ Whichever one we choose, running it we'll see output something like the followin ``` -Trying example: test_floats_are_commutative(x=-0.05851890381391768, y=-6.060045836901702e+300) -Trying example: test_floats_are_commutative(x=-0.06323690311413645, y=2.0324087421708266e-308) -Trying example: test_floats_are_commutative(x=-0.05738038380011458, y=-1.5993500302384265e-308) -Trying example: test_floats_are_commutative(x=-0.06598754758697359, y=-1.1412902232349034e-308) -Trying example: test_floats_are_commutative(x=-0.06472919559855002, y=1.7429441378277974e+35) -Trying example: test_floats_are_commutative(x=-0.06537123121982172, y=-8.136220566134233e-156) -Trying example: test_floats_are_commutative(x=-0.06016703321602157, y=1.9718842567475311e-215) -Trying example: test_floats_are_commutative(x=-0.055257588875432875, y=1.578407827448836e-308) -Trying example: test_floats_are_commutative(x=-0.06313031758042666, y=1.6749023021600297e-175) -Trying example: test_floats_are_commutative(x=-0.05886897920547916, y=1.213699633272585e+292) -Trying example: test_floats_are_commutative(x=-12.0, y=-0.0) -Trying example: test_floats_are_commutative(x=4.0, y=1.7976931348623157e+308) -Trying example: test_floats_are_commutative(x=-9.0, y=0.0) -Trying example: test_floats_are_commutative(x=-38.0, y=1.7976931348623157e+308) -Trying example: test_floats_are_commutative(x=-24.0, y=1.5686642754811104e+289) -Trying example: test_floats_are_commutative(x=-10.0, y=nan) +Test case: test_floats_are_commutative(x=-0.05851890381391768, y=-6.060045836901702e+300) +Test case: test_floats_are_commutative(x=-0.06323690311413645, y=2.0324087421708266e-308) +Test case: test_floats_are_commutative(x=-0.05738038380011458, y=-1.5993500302384265e-308) +Test case: test_floats_are_commutative(x=-0.06598754758697359, y=-1.1412902232349034e-308) +Test case: test_floats_are_commutative(x=-0.06472919559855002, y=1.7429441378277974e+35) +Test case: test_floats_are_commutative(x=-0.06537123121982172, y=-8.136220566134233e-156) +Test case: test_floats_are_commutative(x=-0.06016703321602157, y=1.9718842567475311e-215) +Test case: test_floats_are_commutative(x=-0.055257588875432875, y=1.578407827448836e-308) +Test case: test_floats_are_commutative(x=-0.06313031758042666, y=1.6749023021600297e-175) +Test case: test_floats_are_commutative(x=-0.05886897920547916, y=1.213699633272585e+292) +Test case: test_floats_are_commutative(x=-12.0, y=-0.0) +Test case: test_floats_are_commutative(x=4.0, y=1.7976931348623157e+308) +Test case: test_floats_are_commutative(x=-9.0, y=0.0) +Test case: test_floats_are_commutative(x=-38.0, y=1.7976931348623157e+308) +Test case: test_floats_are_commutative(x=-24.0, y=1.5686642754811104e+289) +Test case: test_floats_are_commutative(x=-10.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (-10.0 + nan) == (nan + -10.0) -Trying example: test_floats_are_commutative(x=10.0, y=nan) +Test case: test_floats_are_commutative(x=10.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (10.0 + nan) == (nan + 10.0) -Trying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (0.0 + nan) == (nan + 0.0) -Trying example: test_floats_are_commutative(x=0.0, y=0.0) -Trying example: test_floats_are_commutative(x=0.0, y=inf) -Trying example: test_floats_are_commutative(x=0.0, y=-inf) -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=0.0) +Test case: test_floats_are_commutative(x=0.0, y=inf) +Test case: test_floats_are_commutative(x=0.0, y=-inf) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` Notice how the first failing example we got was -10.0, nan but Hypothesis was able @@ -121,30 +121,30 @@ Hypothesis's output easy to understand. ``` -Trying example: test_floats_are_commutative(x=nan, y=0.0) +Test case: test_floats_are_commutative(x=nan, y=0.0) Traceback (most recent call last): ... AssertionError: assert (nan + 0.0) == (0.0 + nan) -Trying example: test_floats_are_commutative(x=0.0, y=0.0) -Trying example: test_floats_are_commutative(x=inf, y=0.0) -Trying example: test_floats_are_commutative(x=-inf, y=0.0) -Falsifying example: test_floats_are_commutative(x=nan, y=0.0) +Test case: test_floats_are_commutative(x=0.0, y=0.0) +Test case: test_floats_are_commutative(x=inf, y=0.0) +Test case: test_floats_are_commutative(x=-inf, y=0.0) +Failing test case: test_floats_are_commutative(x=nan, y=0.0) ``` Now lets see what happens when we rerun the test: ``` -Trying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (0.0 + nan) == (nan + 0.0) -Trying example: test_floats_are_commutative(x=0.0, y=0.0) -Trying example: test_floats_are_commutative(x=0.0, y=inf) -Trying example: test_floats_are_commutative(x=0.0, y=-inf) -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=0.0) +Test case: test_floats_are_commutative(x=0.0, y=inf) +Test case: test_floats_are_commutative(x=0.0, y=-inf) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` Notice how the first example it tried was the failing example we had last time? That's @@ -171,11 +171,11 @@ def test_floats_are_commutative(x, y): ``` ``` -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` If you run this in verbose mode it will print out -Falsifying example: test\_floats\_are\_commutative(x=0.0, y=nan) immediately and +Failing test case: test\_floats\_are\_commutative(x=0.0, y=nan) immediately and not try to do any shrinks. Values you pass in via example will not be shrunk. This is partly a technical limitation but it can often be useful as well. diff --git a/website/content/2016-04-16-encode-decode-invariant.md b/website/content/2016-04-16-encode-decode-invariant.md index 0a1cec1fe9..400b878cd1 100644 --- a/website/content/2016-04-16-encode-decode-invariant.md +++ b/website/content/2016-04-16-encode-decode-invariant.md @@ -70,7 +70,7 @@ fuzzing: The code does not correctly handle the empty string. ``` -Falsifying example: test_decode_inverts_encode(s='') +Failing test case: test_decode_inverts_encode(s='') UnboundLocalError: local variable 'character' referenced before assignment ``` @@ -125,7 +125,7 @@ E + 110 test_encoding.py:35: AssertionError ------------------------------------ Hypothesis ------------------------------------ -Falsifying example: test_decode_inverts_encode(s='110') +Failing test case: test_decode_inverts_encode(s='110') ``` diff --git a/website/content/2016-04-19-rule-based-stateful-testing.md b/website/content/2016-04-19-rule-based-stateful-testing.md index 8a2c009178..22cebc6294 100644 --- a/website/content/2016-04-19-rule-based-stateful-testing.md +++ b/website/content/2016-04-19-rule-based-stateful-testing.md @@ -86,7 +86,7 @@ E Use -v to get the full diff binheap.py:74: AssertionError ----- Hypothesis ----- -Falsifying example: test_pop_in_sorted_order(ls=[0, 1, 0]) +Failing test case: test_pop_in_sorted_order(ls=[0, 1, 0]) ``` So we replace heappop with a correct implementation which rebalances the heap: diff --git a/website/content/2016-04-29-testing-performance-optimizations.md b/website/content/2016-04-29-testing-performance-optimizations.md index 36d8a98dc3..c803de2c1d 100644 --- a/website/content/2016-04-29-testing-performance-optimizations.md +++ b/website/content/2016-04-29-testing-performance-optimizations.md @@ -95,7 +95,7 @@ E Use -v to get the full diff foo.py:43: AssertionError ----- Hypothesis ----- -Falsifying example: test_bubble_sorting_is_same_as_merge_sorting(ls=[0, 0]) +Failing test case: test_bubble_sorting_is_same_as_merge_sorting(ls=[0, 0]) ``` What's happened is that we messed up our implementation of merge\_sorted\_lists, because we forgot diff --git a/website/content/2016-05-29-testing-optimizers-with-hypothesis.md b/website/content/2016-05-29-testing-optimizers-with-hypothesis.md index 85a28aea6c..1958c36684 100644 --- a/website/content/2016-05-29-testing-optimizers-with-hypothesis.md +++ b/website/content/2016-05-29-testing-optimizers-with-hypothesis.md @@ -123,7 +123,7 @@ In fact, both of these tests fail: ``` -Falsifying example: test_cloning_an_item(items=[(1, 1), (1, 1), (2, 5)], capacity=7, data=data(...)) +Failing test case: test_cloning_an_item(items=[(1, 1), (1, 1), (2, 5)], capacity=7, data=data(...)) Draw 1: (1, 1) ``` @@ -135,7 +135,7 @@ items that are small enough to fit in it. ``` -Falsifying example: test_removing_a_chosen_item(items=[(1, 1), (2, 4), (1, 2)], capacity=6, data=data(...)) +Failing test case: test_removing_a_chosen_item(items=[(1, 1), (2, 4), (1, 2)], capacity=6, data=data(...)) Draw 1: (1, 1) ``` diff --git a/website/content/2016-06-13-testing-configuration-parameters.md b/website/content/2016-06-13-testing-configuration-parameters.md index e0f905cee5..ff5307437c 100644 --- a/website/content/2016-06-13-testing-configuration-parameters.md +++ b/website/content/2016-06-13-testing-configuration-parameters.md @@ -126,14 +126,14 @@ In both cases, a password would no longer validate against itself: ``` -Falsifying example: test_a_password_verifies( +Failing test case: test_a_password_verifies( password='', time_cost=1, parallelism=1, memory_cost=8, hash_len=4, salt_len=8, ) ``` ``` -Falsifying example: test_a_password_verifies( +Failing test case: test_a_password_verifies( password='', time_cost=1, parallelism=1, memory_cost=8, hash_len=513, salt_len=8 ) diff --git a/website/content/2016-06-30-tests-as-complete-specifications.md b/website/content/2016-06-30-tests-as-complete-specifications.md index c34834a6ea..e972d275ca 100644 --- a/website/content/2016-06-30-tests-as-complete-specifications.md +++ b/website/content/2016-06-30-tests-as-complete-specifications.md @@ -103,7 +103,7 @@ And sure enough, if you run the test enough times it eventually *does* fail: ``` -Falsifying example: test_inserting_at_smaller_index_gives_unsorted( +Failing test case: test_inserting_at_smaller_index_gives_unsorted( ls=[0, 1, 1, 1, 1], v=1 ) ``` diff --git a/website/content/2016-07-04-calculating-the-mean.md b/website/content/2016-07-04-calculating-the-mean.md index 7e45f0cad6..76e08c4abd 100644 --- a/website/content/2016-07-04-calculating-the-mean.md +++ b/website/content/2016-07-04-calculating-the-mean.md @@ -51,7 +51,7 @@ assert inf <= 8.98846567431158e+307 + where inf = mean([8.988465674311579e+307, 8.98846567431158e+307]) + and 8.98846567431158e+307 = max([8.988465674311579e+307, 8.98846567431158e+307]) -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[8.988465674311579e+307, 8.98846567431158e+307] ) ``` @@ -74,7 +74,7 @@ assert 1.390671161567e-309 <= 1.390671161566996e-309 where 1.390671161567e-309 = min([1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309]) and 1.390671161566996e-309 = mean([1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309]) -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309] ) ``` @@ -107,7 +107,7 @@ assert inf <= 8.98846567431158e+307 where inf = mean([8.988465674311579e+307, 8.98846567431158e+307]) and 8.98846567431158e+307 = max([8.988465674311579e+307, 8.98846567431158e+307]) -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[8.988465674311579e+307, 8.98846567431158e+307] ) ``` @@ -119,7 +119,7 @@ this is broken too ``` OverflowError: integer division result too large for a float -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[8.988465674311579e+307, 8.98846567431158e+307] ) ``` diff --git a/website/content/2016-07-23-what-is-hypothesis.md b/website/content/2016-07-23-what-is-hypothesis.md index 1b4377f752..644e541f00 100644 --- a/website/content/2016-07-23-what-is-hypothesis.md +++ b/website/content/2016-07-23-what-is-hypothesis.md @@ -90,7 +90,7 @@ When this is first run, you will see an error that looks something like this: ``` -Falsifying example: test_decode_inverts_encode(s='\xc2\xc2\x80') +Failing test case: test_decode_inverts_encode(s='\xc2\xc2\x80') Traceback (most recent call last): File "/home/david/.pyenv/versions/2.7/lib/python2.7/site-packages/hypothesis/core.py", line 443, in evaluate_test_data diff --git a/website/content/2017-03-09-hypothesis-for-researchers.md b/website/content/2017-03-09-hypothesis-for-researchers.md index c87c80715b..bc1739fc60 100644 --- a/website/content/2017-03-09-hypothesis-for-researchers.md +++ b/website/content/2017-03-09-hypothesis-for-researchers.md @@ -99,7 +99,7 @@ Then on running we would see the following output: ---- Hypothesis ---- - Falsifying example: test_sort_is_idempotent(ls=[0, 1]) + Failing test case: test_sort_is_idempotent(ls=[0, 1]) ``` Hypothesis probably started with a much more complicated example (the test fails for essentially any list with more @@ -135,7 +135,7 @@ This fails because we've forgotten than `i` may be zero, and also about Python's ---- Hypothesis ---- - Falsifying example: test_sort_is_idempotent(ls=[0, 1], data=data(...)) + Failing test case: test_sort_is_idempotent(ls=[0, 1], data=data(...)) Draw 1: 0 ``` diff --git a/website/content/2017-09-14-multi-bug-discovery.md b/website/content/2017-09-14-multi-bug-discovery.md index 52a49e5b74..9ba7829867 100644 --- a/website/content/2017-09-14-multi-bug-discovery.md +++ b/website/content/2017-09-14-multi-bug-discovery.md @@ -63,7 +63,7 @@ Well, as of Hypothesis 3.29.0, released a few weeks ago, now it does! If you run the above test now, you'll get the following: ``` -Falsifying example: test(ls=[nan]) +Failing test case: test(ls=[nan]) Traceback (most recent call last): File "/home/david/hypothesis-python/src/hypothesis/core.py", line 671, in run print_example=True, is_final=True @@ -79,7 +79,7 @@ Traceback (most recent call last): assert min(ls) <= mean(ls) <= max(ls) AssertionError -Falsifying example: test(ls=[]) +Failing test case: test(ls=[]) Traceback (most recent call last): File "/home/david/hypothesis-python/src/hypothesis/core.py", line 671, in run print_example=True, is_final=True diff --git a/website/content/2018-01-08-smarkets.md b/website/content/2018-01-08-smarkets.md index 1215455603..ac2e683b03 100644 --- a/website/content/2018-01-08-smarkets.md +++ b/website/content/2018-01-08-smarkets.md @@ -121,7 +121,7 @@ Over all I'm much happier with the new health check system and think it does a m Historically output from Hypothesis has looked something like this: ``` -Falsifying example: test_is_minimal(ls=[0], v=1) +Failing test case: test_is_minimal(ls=[0], v=1) ``` Or, if you had a failed health check, like the following: