Make flaky test reruns draw fresh random values (#494)

pytest-randomly reseeds the global RNG at the start of every test call
phase, including reruns triggered by the flaky plugin. A randomized test
that fails for the session seed therefore fails identically on all
retries, making @flaky.flaky useless and causing spurious CI failures.

Add a fresh_seed_each_run decorator that mixes the rerun attempt number
into the seed inside the test call (a fixture cannot do this, since
pytest-randomly reseeds after fixture setup), apply it to all flaky
tests, and add retries to the scaling-invariance test which had none.

Author: basnijholt

adaptive/tests/flaky_utils.py

@@ -0,0 +1,39 @@
+"""Make ``flaky`` reruns work with ``pytest-randomly``."""
+
+import functools as ft
+import random
+from collections import Counter
+
+import numpy as np
+
+_attempts: Counter = Counter()
+
+
+def fresh_seed_each_run(func):
+    """Make ``@flaky.flaky`` reruns draw new random values.
+
+    ``pytest-randomly`` reseeds the global RNG at the start of every test
+    call phase — including reruns triggered by the ``flaky`` plugin — so a
+    randomized test that fails for the session seed fails identically on
+    every rerun, making the retries useless. Reseeding cannot happen in a
+    fixture (``pytest-randomly`` reseeds in ``pytest_runtest_call``, after
+    fixture setup), so this mixes the attempt number into the seed at the
+    start of the test call itself. Each rerun gets new draws while the
+    whole sequence stays reproducible via ``--randomly-seed``.
+
+    Apply directly on the test function, below ``@flaky.flaky`` and any
+    parametrization.
+    """
+
+    @ft.wraps(func)
+    def wrapper(*args, **kwargs):
+        key = (func.__qualname__, repr(args), repr(kwargs))
+        attempt = _attempts[key]
+        _attempts[key] += 1
+        if attempt:
+            seed = (random.getrandbits(32) + attempt) % 2**32
+            random.seed(seed)
+            np.random.seed(seed)
+        return func(*args, **kwargs)
+
+    return wrapper

adaptive/tests/test_average_learner.py

@@ -5,6 +5,7 @@
 
 from adaptive.learner import AverageLearner
 from adaptive.runner import simple
+from adaptive.tests.flaky_utils import fresh_seed_each_run
 
 
 def f_unused(seed):
@@ -28,6 +29,7 @@ def test_only_returns_new_points():
 
 
 @flaky.flaky(max_runs=5)
+@fresh_seed_each_run
 def test_avg_std_and_npoints():
     learner = AverageLearner(f_unused, atol=None, rtol=0.01)
 

adaptive/tests/test_learner1d.py

@@ -9,6 +9,7 @@
 from adaptive.learner import Learner1D
 from adaptive.learner.learner1D import curvature_loss_function
 from adaptive.runner import BlockingRunner, simple
+from adaptive.tests.flaky_utils import fresh_seed_each_run
 
 
 def flat_middle(x):
@@ -259,6 +260,7 @@ def test_ask_does_not_return_known_points_when_returning_bounds():
 
 
 @flaky.flaky(max_runs=3)
+@fresh_seed_each_run
 def test_tell_many():
     def f(x, offset=0.123214):
         a = 0.01

adaptive/tests/test_learners.py

@@ -29,6 +29,7 @@
 )
 from adaptive.learner.learner1D import with_pandas
 from adaptive.runner import simple
+from adaptive.tests.flaky_utils import fresh_seed_each_run
 
 LOSS_FUNCTIONS = {
     Learner1D: (
@@ -514,7 +515,9 @@ def test_expected_loss_improvement_is_less_than_total_loss(
 
 # XXX: This *should* pass (https://github.com/python-adaptive/adaptive/issues/55)
 #      but we xfail it now, as Learner2D will be deprecated anyway
+@flaky.flaky(max_runs=5)
 @run_with(Learner1D, xfail(Learner2D), LearnerND, AverageLearner1D)
+@fresh_seed_each_run
 def test_learner_performance_is_invariant_under_scaling(
     learner_type, f, learner_kwargs
 ):
@@ -583,6 +586,7 @@ def scale_x(x):
     SequenceLearner,
     with_all_loss_functions=False,
 )
+@fresh_seed_each_run
 def test_balancing_learner(learner_type, f, learner_kwargs):
     """Test if the BalancingLearner works with the different types of learners."""
     learners = [