Skip to content

Commit b86d7c2

Browse files
arettigmhucka
andauthored
Rerun flaky test in vpe_circuits_test.py (#1339)
This test was causing intermittent failures of the CI in the 5% of runs where the measured value is outside 2 standard deviations from the expected value. The built in `@cirq.testing.retry_once_with_later_random_values` decorator will now rerun the test with a new random seed. --------- Co-authored-by: mhucka <mhucka@google.com>
1 parent b69bbbb commit b86d7c2

4 files changed

Lines changed: 115 additions & 1 deletion

File tree

src/openfermion/circuits/vpe_circuits_test.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import cirq
1616

1717
from openfermion.measurements import get_phase_function
18+
from openfermion.testing import retry_once_with_later_random_values
1819

1920
from .vpe_circuits import vpe_single_circuit, vpe_circuits_single_timestep
2021

@@ -35,6 +36,7 @@ def test_single_circuit():
3536
assert data_counts[1] == 100
3637

3738

39+
@retry_once_with_later_random_values
3840
def test_single_timestep():
3941
q0 = cirq.GridQubit(0, 0)
4042
q1 = cirq.GridQubit(0, 1)

src/openfermion/testing/__init__.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,8 @@
2626
module_importable,
2727
)
2828

29-
from .wrapped import assert_equivalent_repr, assert_implements_consistent_protocols
29+
from .wrapped import (
30+
assert_equivalent_repr,
31+
assert_implements_consistent_protocols,
32+
retry_once_with_later_random_values,
33+
)

src/openfermion/testing/wrapped.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,28 @@ def assert_implements_consistent_protocols(
6565
global_vals=global_vals, # coverage: ignore
6666
local_vals=local_vals, # coverage: ignore
6767
) # coverage: ignore
68+
69+
70+
def retry_once_with_later_random_values(testfunc: Any) -> Any:
71+
"""Marks a test function for one retry with later random values.
72+
73+
This decorator is intended for test functions which occasionally fail
74+
for specific random seeds from pytest-randomly.
75+
"""
76+
try:
77+
return cirq.testing.retry_once_with_later_random_values(testfunc)
78+
except AttributeError:
79+
# decorator not available in cirq < 1.5.0
80+
import functools
81+
import warnings
82+
83+
@functools.wraps(testfunc)
84+
def wrapped_func(*args, **kwargs) -> Any:
85+
try:
86+
return testfunc(*args, **kwargs)
87+
except AssertionError:
88+
pass
89+
warnings.warn("Retrying in case we got a failing seed from pytest-randomly.")
90+
return testfunc(*args, **kwargs)
91+
92+
return wrapped_func
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pytest
16+
from unittest import mock
17+
import openfermion.testing.wrapped as wrapped
18+
19+
20+
class MockCirqTesting:
21+
@property
22+
def retry_once_with_later_random_values(self):
23+
raise AttributeError("No such attribute")
24+
25+
26+
class MockCirq:
27+
testing = MockCirqTesting()
28+
29+
30+
def test_retry_once_fallback_success():
31+
with mock.patch('openfermion.testing.wrapped.cirq', MockCirq()):
32+
33+
call_count = 0
34+
35+
@wrapped.retry_once_with_later_random_values
36+
def successful_test():
37+
nonlocal call_count
38+
call_count += 1
39+
return "Success"
40+
41+
assert successful_test() == "Success"
42+
assert call_count == 1
43+
44+
45+
def test_retry_once_fallback_flaky():
46+
with mock.patch('openfermion.testing.wrapped.cirq', MockCirq()):
47+
48+
call_count = 0
49+
50+
@wrapped.retry_once_with_later_random_values
51+
def flaky_test():
52+
nonlocal call_count
53+
call_count += 1
54+
if call_count == 1:
55+
raise AssertionError("Failed first time")
56+
return "Success"
57+
58+
with pytest.warns(
59+
UserWarning, match="Retrying in case we got a failing seed from pytest-randomly."
60+
):
61+
assert flaky_test() == "Success"
62+
63+
assert call_count == 2
64+
65+
66+
def test_retry_once_fallback_failure():
67+
with mock.patch('openfermion.testing.wrapped.cirq', MockCirq()):
68+
69+
call_count = 0
70+
71+
@wrapped.retry_once_with_later_random_values
72+
def failing_test():
73+
nonlocal call_count
74+
call_count += 1
75+
raise AssertionError("Failed both times")
76+
77+
with pytest.warns(
78+
UserWarning, match="Retrying in case we got a failing seed from pytest-randomly."
79+
):
80+
with pytest.raises(AssertionError, match="Failed both times"):
81+
failing_test()
82+
83+
assert call_count == 2

0 commit comments

Comments
 (0)