test: stabilize flaky tests (getsentry#1614)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: jpnurmi

.github/workflows/ci.yml

@@ -364,8 +364,11 @@ jobs:
 
       - name: Install sccache
         if: ${{ runner.os == 'Windows' && env['USE_SCCACHE'] }}
+        continue-on-error: true # build with plain ninja if chocolatey is down
         shell: bash
-        run: choco install sccache -y
+        run: |
+          choco install sccache -y
+          sccache --version
 
       - name: Cache sccache
         if: ${{ runner.os == 'Windows' && env['USE_SCCACHE'] }}

tests/__init__.py

@@ -12,6 +12,13 @@
 
 sourcedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
+
+def adb(*args, **kwargs):
+    return subprocess.run(
+        ["{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]), *args], **kwargs
+    )
+
+
 # https://docs.pytest.org/en/latest/assert.html#assert-details
 pytest.register_assert_rewrite("tests.assertions")
 

tests/assertions.py

@@ -5,9 +5,11 @@
 import re
 import sys
 from dataclasses import dataclass
-from datetime import datetime, UTC
+from datetime import datetime, timedelta, UTC
 from pathlib import Path
 
+import tests
+
 import msgpack
 
 from . import SENTRY_VERSION
@@ -360,8 +362,10 @@ def assert_minidump(envelope):
 
 
 def assert_timestamp(ts):
-    elapsed_time = datetime.now(UTC) - datetime.fromisoformat(ts)
-    assert elapsed_time.total_seconds() < 10
+    dt = datetime.fromisoformat(ts)
+    # 1s tolerance for `date +%s` truncation in device clock offset measurement
+    assert dt <= tests.now() + timedelta(seconds=1), "timestamp is in the future"
+    assert dt >= tests.test_start, "timestamp is in the past"
 
 
 def assert_event(envelope, message="Hello World!", expected_trace_id=""):

tests/cmake.py

@@ -8,6 +8,8 @@
 
 import pytest
 
+from . import adb
+from .conditions import has_sccache
 from .build_config import (
     get_android_config,
     get_platform_cmake_args,
@@ -129,7 +131,7 @@ def cmake_configure(cwd, options, cflags=None):
     __tracebackhide__ = True
 
     options = dict(options)
-    if os.environ.get("USE_SCCACHE"):
+    if has_sccache:
         options.update(
             {
                 "CMAKE_C_COMPILER_LAUNCHER": "sccache",
@@ -233,7 +235,7 @@ def cmake_build(cwd, targets, options):
             "cmake",
         ]
     env = dict(os.environ)
-    if env.get("USE_SCCACHE"):
+    if has_sccache:
         # Each pytest run builds in a new temp directory. Paths are normalized
         # relative to the build dir to allow sccache hits across runs.
         env.setdefault("SCCACHE_BASEDIRS", str(cwd))
@@ -308,16 +310,7 @@ def cmake_build(cwd, targets, options):
 
     if os.environ.get("ANDROID_API"):
         # copy the output to the android image via adb
-        subprocess.run(
-            [
-                "{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]),
-                "push",
-                "./",
-                "/data/local/tmp",
-            ],
-            cwd=cwd,
-            check=True,
-        )
+        adb("push", "./", "/data/local/tmp", cwd=cwd, check=True)
 
 
 def configure_clang_cl(config_cmd: list[str]):

tests/conditions.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import shutil
 
 is_aix = sys.platform == "aix" or sys.platform == "os400"
 is_android = os.environ.get("ANDROID_API")
@@ -48,3 +49,5 @@
 # It's always available - tests explicitly set SENTRY_BACKEND: native in cmake
 # On macOS ASAN, the signal handling conflicts with ASAN's memory interception
 has_native = has_http and not (is_asan and sys.platform == "darwin")
+
+has_sccache = os.environ.get("USE_SCCACHE") and shutil.which("sccache")

tests/conftest.py

@@ -5,8 +5,11 @@
 import re
 import statistics
 import sys
-from . import run
+from datetime import datetime, timedelta, UTC
+from . import adb, run
 from .cmake import CMake
+from .conditions import has_sccache, is_android
+import tests
 
 LABEL = "label"
 TIME_UNIT = "time_unit"
@@ -41,6 +44,28 @@ def cmake(tmp_path_factory):
     cmake.destroy()
 
 
+def _get_clock_offset():
+    """Measure clock offset between host and Android device."""
+    if not is_android:
+        return timedelta(0)
+    try:
+        before = datetime.now(UTC)
+        result = adb("shell", "date", "+%s", capture_output=True, text=True)
+        after = datetime.now(UTC)
+        device_time = datetime.fromtimestamp(int(result.stdout.strip()), tz=UTC)
+        host_time = before + (after - before) / 2
+        return device_time - host_time
+    except (KeyError, ValueError, OSError):
+        return timedelta(0)
+
+
+@pytest.fixture(autouse=True)
+def _record_test_start():
+    offset = _get_clock_offset()
+    tests.now = lambda: datetime.now(UTC) + offset
+    tests.test_start = tests.now()
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--with_crashpad_wer",
@@ -143,12 +168,12 @@ def pytest_sessionfinish(session, exitstatus):
 
 
 def pytest_sessionstart(session):
-    if os.environ.get("USE_SCCACHE"):
+    if has_sccache:
         subprocess.run(["sccache", "--zero-stats"], capture_output=True)
 
 
 def pytest_terminal_summary(terminalreporter):
-    if os.environ.get("USE_SCCACHE"):
+    if has_sccache:
         result = subprocess.run(
             ["sccache", "--show-stats"], capture_output=True, text=True
         )

tests/test_dotnet_signals.py

@@ -7,6 +7,7 @@
 
 import pytest
 
+from tests import adb
 from tests.conditions import is_android, is_tsan, is_x86, is_asan
 
 project_fixture_path = pathlib.Path("tests/fixtures/dotnet_signal")
@@ -275,11 +276,6 @@ def wait_for(condition, timeout=10, interval=0.5):
     return condition()
 
 
-def adb(*args, **kwargs):
-    adb_path = "{}/platform-tools/adb".format(os.environ["ANDROID_HOME"])
-    return subprocess.run([adb_path, *args], **kwargs)
-
-
 def run_android(args=None, timeout=30):
     if args is None:
         args = []

tests/test_inproc_stress.py

@@ -8,29 +8,16 @@
 
 import pytest
 
-from . import Envelope
+from . import adb, Envelope
 from .assertions import assert_inproc_crash
 from .build_config import get_test_executable_cmake_args, get_test_executable_env
-from .conditions import is_tsan
+from .conditions import is_android, is_tsan
 
 fixture_path = pathlib.Path("tests/fixtures/inproc_stress")
 
 ANDROID_TMP = "/data/local/tmp"
 
 
-def is_android():
-    return bool(os.environ.get("ANDROID_API"))
-
-
-def adb(*args):
-    """Run an adb command."""
-    return subprocess.run(
-        ["{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]), *args],
-        check=True,
-        capture_output=True,
-    )
-
-
 def compile_test_program(tmp_path):
     build_dir = tmp_path / "inproc_stress_build"
     build_dir.mkdir(exist_ok=True)
@@ -63,14 +50,14 @@ def compile_test_program(tmp_path):
     exe_path = build_dir / exe_name
 
     # Push executable to Android device
-    if is_android():
-        adb("push", str(exe_path), ANDROID_TMP)
+    if is_android:
+        adb("push", str(exe_path), ANDROID_TMP, check=True, capture_output=True)
 
     return exe_path
 
 
 def run_stress_test(tmp_path, test_executable, test_name, database_path=None):
-    if is_android():
+    if is_android:
         return run_stress_test_android(test_executable, test_name, database_path)
 
     if database_path is None:
@@ -100,20 +87,14 @@ def run_stress_test_android(test_executable, test_name, database_path):
     remote_db_path = f"{ANDROID_TMP}/{database_path.name}"
 
     # Clear logcat before running so we limit the capture as close to this run as possible
-    subprocess.run(
-        ["{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]), "logcat", "-c"],
-        check=False,
-    )
+    adb("logcat", "-c", check=False)
 
     # Run on device - we need to capture both stdout and stderr, and the return code
     # Android shell doesn't separate stdout/stderr well, so we redirect stderr to stdout
     # and parse the return code from the output (same approach as tests/__init__.py)
-    result = subprocess.run(
-        [
-            "{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]),
-            "shell",
-            f"cd {ANDROID_TMP} && LD_LIBRARY_PATH=. ./{exe_name} {test_name} {remote_db_path} 2>&1; echo ret:$?",
-        ],
+    result = adb(
+        "shell",
+        f"cd {ANDROID_TMP} && LD_LIBRARY_PATH=. ./{exe_name} {test_name} {remote_db_path} 2>&1; echo ret:$?",
         capture_output=True,
         text=True,
     )
@@ -130,14 +111,11 @@ def run_stress_test_android(test_executable, test_name, database_path):
     time.sleep(0.5)
 
     # Capture logcat to get our logs
-    logcat_result = subprocess.run(
-        [
-            "{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]),
-            "logcat",
-            "-d",
-            "-s",
-            "sentry-native:*",
-        ],
+    logcat_result = adb(
+        "logcat",
+        "-d",
+        "-s",
+        "sentry-native:*",
         capture_output=True,
         text=True,
     )
@@ -150,20 +128,19 @@ def run_stress_test_android(test_executable, test_name, database_path):
 
     # Pull the remote database to local path (pulls to parent, creates database_path)
     try:
-        adb("pull", f"{remote_db_path}/", str(database_path.parent))
+        adb(
+            "pull",
+            f"{remote_db_path}/",
+            str(database_path.parent),
+            check=True,
+            capture_output=True,
+        )
     except subprocess.CalledProcessError:
         # Database might not exist if crash wasn't captured
         pass
 
     # Clean up remote database for next run
-    subprocess.run(
-        [
-            "{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]),
-            "shell",
-            f"rm -rf {remote_db_path}",
-        ],
-        check=False,
-    )
+    adb("shell", f"rm -rf {remote_db_path}", check=False)
 
     # Combine shell output with logcat output for assertion checks
     combined_output = output + "\n" + logcat_output
@@ -374,7 +351,7 @@ def test_inproc_handler_abort_crash(cmake):
 
 
 @pytest.mark.skipif(
-    sys.platform != "darwin" or is_android(),
+    sys.platform != "darwin" or bool(is_android),
     reason="Stack trace tests are macOS-only",
 )
 @pytest.mark.parametrize(

tests/test_integration_native.py

@@ -99,24 +99,26 @@ def test_native_oom(cmake, httpserver):
     """Test OOM crash capture with native backend"""
     tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "native"})
 
-    httpserver.expect_request("/api/123456/envelope/").respond_with_data("OK")
+    httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK")
 
-    run_crash(
-        tmp_path,
-        "sentry_example",
-        ["log", "stdout", "oom"],
-        env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)),
-    )
+    with httpserver.wait(timeout=10) as waiting:
+        run_crash(
+            tmp_path,
+            "sentry_example",
+            ["log", "stdout", "oom"],
+            env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)),
+        )
 
-    time.sleep(2)
+        time.sleep(2)
 
-    run(
-        tmp_path,
-        "sentry_example",
-        ["log", "no-setup"],
-        env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)),
-    )
+        run(
+            tmp_path,
+            "sentry_example",
+            ["log", "no-setup"],
+            env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)),
+        )
 
+    assert waiting.result
     assert len(httpserver.log) >= 1