Python(feat): pytest graceful handling missing connection (#569)

Author: alexluck-sift

python/docs/examples/pytest_plugin.md

@@ -3,7 +3,7 @@
 import logging
 import uuid
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from google.protobuf import json_format
 from sift.test_reports.v1.test_reports_pb2 import (
@@ -68,6 +68,9 @@
 logger = logging.getLogger(__name__)
 
 
+_EntityT = TypeVar("_EntityT", TestReport, TestStep, TestMeasurement)
+
+
 class TestResultsLowLevelClient(LowLevelClientBase, WithGrpcClient):
     """Low-level client for the TestResultsAPI.
 
@@ -82,6 +85,16 @@ def __init__(self, grpc_client: GrpcClient):
         """
         super().__init__(grpc_client)
 
+    @staticmethod
+    def _mark_simulated(instance: _EntityT) -> _EntityT:
+        """Stamp an entity as having been produced by the simulate path.
+
+        Mirrors the ``__dict__`` write used by ``BaseType._apply_client_to_instance``
+        to bypass pydantic's frozen-model guard.
+        """
+        instance.__dict__["_simulated"] = True
+        return instance
+
     @staticmethod
     def simulate_create_test_report_response(
         request: CreateTestReportRequest,
@@ -387,7 +400,7 @@ async def create_test_report(
                     request,
                     response_id=simulated_proto.test_report_id,
                 )
-            return TestReport._from_proto(simulated_proto)
+            return self._mark_simulated(TestReport._from_proto(simulated_proto))
 
         response = await self._grpc_client.get_stub(TestReportServiceStub).CreateTestReport(request)
         grpc_test_report = cast("CreateTestReportResponse", response).test_report
@@ -505,7 +518,9 @@ async def update_test_report(
         if log_file is not None or simulate:
             if log_file is not None:
                 log_request_to_file(log_file, "UpdateTestReport", request)
-            return self.simulate_update_test_report_response(request, existing=existing)
+            return self._mark_simulated(
+                self.simulate_update_test_report_response(request, existing=existing)
+            )
 
         response = await self._grpc_client.get_stub(TestReportServiceStub).UpdateTestReport(request)
         grpc_test_report = cast("UpdateTestReportResponse", response).test_report
@@ -560,7 +575,7 @@ async def create_test_step(
                     request,
                     response_id=simulated_proto.test_step_id,
                 )
-            return TestStep._from_proto(simulated_proto)
+            return self._mark_simulated(TestStep._from_proto(simulated_proto))
 
         response = await self._grpc_client.get_stub(TestReportServiceStub).CreateTestStep(request)
         grpc_test_step = cast("CreateTestStepResponse", response).test_step
@@ -661,7 +676,9 @@ async def update_test_step(
         if log_file is not None or simulate:
             if log_file is not None:
                 log_request_to_file(log_file, "UpdateTestStep", request)
-            return self.simulate_update_test_step_response(request, existing=existing)
+            return self._mark_simulated(
+                self.simulate_update_test_step_response(request, existing=existing)
+            )
 
         response = await self._grpc_client.get_stub(TestReportServiceStub).UpdateTestStep(request)
         grpc_test_step = cast("UpdateTestStepResponse", response).test_step
@@ -716,7 +733,7 @@ async def create_test_measurement(
                     request,
                     response_id=simulated_proto.measurement_id,
                 )
-            return TestMeasurement._from_proto(simulated_proto)
+            return self._mark_simulated(TestMeasurement._from_proto(simulated_proto))
 
         response = await self._grpc_client.get_stub(TestReportServiceStub).CreateTestMeasurement(
             request
@@ -861,7 +878,9 @@ async def update_test_measurement(
         if log_file is not None or simulate:
             if log_file is not None:
                 log_request_to_file(log_file, "UpdateTestMeasurement", request)
-            return self.simulate_update_test_measurement_response(request, existing=existing)
+            return self._mark_simulated(
+                self.simulate_update_test_measurement_response(request, existing=existing)
+            )
 
         response = await self._grpc_client.get_stub(TestReportServiceStub).UpdateTestMeasurement(
             request

python/lib/sift_client/_internal/low_level_wrappers/test_results.py

@@ -79,5 +79,13 @@ def ci_pytest_tag(sift_client):
 
 
 def pytest_configure(config: pytest.Config) -> None:
-    """Enable the Sift connection-check mode for the fixtures used in this test suite since we run w/ mock client in non-integration tests."""
-    config.option.sift_test_results_check_connection = True
+    """Pick a Sift plugin mode based on whether integration tests are running.
+
+    Integration runs (``-m integration``) stay online with the default
+    log-file pipeline enabled so CI exercises the JSONL write + import
+    worker replay path that production users hit. Every other run defaults
+    to ``--sift-disabled`` so unit tests don't need credentials.
+    """
+    is_integration_run = "integration" in (config.option.markexpr or "")
+    if not is_integration_run:
+        config.option.sift_disabled = True

python/lib/sift_client/_tests/conftest.py

@@ -25,6 +25,15 @@
 
 import pytest
 
+_SIFT_ENV_VARS = ("SIFT_API_KEY", "SIFT_GRPC_URI", "SIFT_REST_URI", "SIFT_DISABLED")
+
+
+@pytest.fixture
+def clear_sift_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Unset all ``SIFT_*`` environment variables for the duration of the test."""
+    for name in _SIFT_ENV_VARS:
+        monkeypatch.delenv(name, raising=False)
+
 
 @pytest.fixture
 def write_plugin_conftest(pytester: pytest.Pytester) -> Callable[[], None]:

python/lib/sift_client/_tests/pytest_plugin/conftest.py

@@ -34,7 +34,7 @@ def test_ini_log_file_none(
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_log_file = "none"
+            sift_log_file = "none"
             """
         )
         pytester.makepyfile("def test_noop(): pass")
@@ -46,7 +46,7 @@ def test_python_false_disables_log_file(
         pytester: pytest.Pytester,
         write_probe_conftest: Callable[[str], None],
     ) -> None:
-        """`config.option.sift_test_results_log_file = False` disables logging.
+        """`config.option.sift_log_file = False` disables logging.
 
         Conftests use this pattern (see lib/sift_client/_tests/util/conftest.py)
         to opt their subtree out of log-file mode. Regression test for the
@@ -55,7 +55,7 @@ def test_python_false_disables_log_file(
         """
         write_probe_conftest(
             """
-            config.option.sift_test_results_log_file = False
+            config.option.sift_log_file = False
             from sift_client.pytest_plugin import _resolve_log_file
             print("RESOLVED:", _resolve_log_file(config))
             """,
@@ -80,33 +80,54 @@ def test_ini_log_file_path(
         pytester.makepyprojecttoml(
             f"""
             [tool.pytest.ini_options]
-            sift_test_results_log_file = "{log_path}"
+            sift_log_file = "{log_path}"
             """
         )
         pytester.makepyfile("def test_noop(): pass")
         result = pytester.runpytest_subprocess("-s", "--co")
         result.stdout.fnmatch_lines([f"RESOLVED: {log_path}"])
 
-    def test_ini_check_connection_true(
+    def test_ini_offline_true(
         self,
         pytester: pytest.Pytester,
         write_probe_conftest: Callable[[str], None],
     ) -> None:
         write_probe_conftest(
             """
-            from sift_client.pytest_plugin import _check_connection_enabled
-            print("CHECK:", _check_connection_enabled(config))
+            from sift_client.pytest_plugin import _is_offline
+            print("OFFLINE:", _is_offline(config))
             """,
         )
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_check_connection = true
+            sift_offline = true
             """
         )
         pytester.makepyfile("def test_noop(): pass")
         result = pytester.runpytest_subprocess("-s", "--co")
-        result.stdout.fnmatch_lines(["CHECK: True"])
+        result.stdout.fnmatch_lines(["OFFLINE: True"])
+
+    def test_ini_disabled_true(
+        self,
+        pytester: pytest.Pytester,
+        write_probe_conftest: Callable[[str], None],
+    ) -> None:
+        write_probe_conftest(
+            """
+            from sift_client.pytest_plugin import _is_disabled
+            print("DISABLED:", _is_disabled(config))
+            """,
+        )
+        pytester.makepyprojecttoml(
+            """
+            [tool.pytest.ini_options]
+            sift_disabled = true
+            """
+        )
+        pytester.makepyfile("def test_noop(): pass")
+        result = pytester.runpytest_subprocess("-s", "--co")
+        result.stdout.fnmatch_lines(["DISABLED: True"])
 
     def test_ini_git_metadata_false(
         self,
@@ -115,13 +136,13 @@ def test_ini_git_metadata_false(
     ) -> None:
         write_probe_conftest(
             """
-            print("INI_GIT:", config.getini("sift_test_results_git_metadata"))
+            print("INI_GIT:", config.getini("sift_git_metadata"))
             """,
         )
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_git_metadata = false
+            sift_git_metadata = false
             """
         )
         pytester.makepyfile("def test_noop(): pass")
@@ -145,49 +166,63 @@ def test_cli_overrides_ini(
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_log_file = "none"
+            sift_log_file = "none"
             """
         )
         pytester.makepyfile("def test_noop(): pass")
-        result = pytester.runpytest_subprocess(
-            "-s", "--co", f"--sift-test-results-log-file={cli_path}"
-        )
+        result = pytester.runpytest_subprocess("-s", "--co", f"--sift-log-file={cli_path}")
         result.stdout.fnmatch_lines([f"RESOLVED: {cli_path}"])
 
-    def test_cli_check_connection_flag(
+    def test_cli_offline_flag(
+        self,
+        pytester: pytest.Pytester,
+        write_probe_conftest: Callable[[str], None],
+    ) -> None:
+        """The ``--sift-offline`` CLI flag flips the resolver to True."""
+        write_probe_conftest(
+            """
+            from sift_client.pytest_plugin import _is_offline
+            print("OFFLINE:", _is_offline(config))
+            """,
+        )
+        pytester.makepyfile("def test_noop(): pass")
+        result = pytester.runpytest_subprocess("-s", "--co", "--sift-offline")
+        result.stdout.fnmatch_lines(["OFFLINE: True"])
+
+    def test_cli_disabled_flag(
         self,
         pytester: pytest.Pytester,
         write_probe_conftest: Callable[[str], None],
     ) -> None:
-        """The ``--sift-test-results-check-connection`` CLI flag flips the resolver to True."""
+        """The ``--sift-disabled`` CLI flag flips the resolver to True."""
         write_probe_conftest(
             """
-            from sift_client.pytest_plugin import _check_connection_enabled
-            print("CHECK:", _check_connection_enabled(config))
+            from sift_client.pytest_plugin import _is_disabled
+            print("DISABLED:", _is_disabled(config))
             """,
         )
         pytester.makepyfile("def test_noop(): pass")
-        result = pytester.runpytest_subprocess("-s", "--co", "--sift-test-results-check-connection")
-        result.stdout.fnmatch_lines(["CHECK: True"])
+        result = pytester.runpytest_subprocess("-s", "--co", "--sift-disabled")
+        result.stdout.fnmatch_lines(["DISABLED: True"])
 
     def test_cli_no_git_metadata_flag(
         self,
         pytester: pytest.Pytester,
         write_probe_conftest: Callable[[str], None],
     ) -> None:
-        """The ``--no-sift-test-results-git-metadata`` CLI flag flips git_metadata to False.
+        """The ``--no-sift-git-metadata`` CLI flag flips git_metadata to False.
 
         Guards the negation flag's ``dest`` binding: the flag name doesn't match
         the ini key, so a broken ``dest`` would silently fall back to the ini
         default and pass every other test in this file.
         """
         write_probe_conftest(
             """
-            print("CLI_GIT:", config.getoption("sift_test_results_git_metadata"))
+            print("CLI_GIT:", config.getoption("sift_git_metadata"))
             """,
         )
         pytester.makepyfile("def test_noop(): pass")
-        result = pytester.runpytest_subprocess("-s", "--co", "--no-sift-test-results-git-metadata")
+        result = pytester.runpytest_subprocess("-s", "--co", "--no-sift-git-metadata")
         result.stdout.fnmatch_lines(["CLI_GIT: False"])
 
     def test_defaults_when_neither_set(
@@ -198,20 +233,23 @@ def test_defaults_when_neither_set(
         write_probe_conftest(
             """
             from sift_client.pytest_plugin import (
-                _check_connection_enabled,
+                _is_disabled,
+                _is_offline,
                 _resolve_log_file,
             )
             print("RESOLVED:", _resolve_log_file(config))
-            print("CHECK:", _check_connection_enabled(config))
-            print("INI_GIT:", config.getini("sift_test_results_git_metadata"))
+            print("OFFLINE:", _is_offline(config))
+            print("DISABLED:", _is_disabled(config))
+            print("INI_GIT:", config.getini("sift_git_metadata"))
             """,
         )
         pytester.makepyfile("def test_noop(): pass")
         result = pytester.runpytest_subprocess("-s", "--co")
         result.stdout.fnmatch_lines(
             [
                 "RESOLVED: True",
-                "CHECK: False",
+                "OFFLINE: False",
+                "DISABLED: False",
                 "INI_GIT: True",
             ]
         )
@@ -238,7 +276,7 @@ def report_context():
 
 
 class TestAutouseGate:
-    """`sift_include` / `sift_exclude` markers and the `sift_test_results_autouse` ini gate."""
+    """`sift_include` / `sift_exclude` markers and the `sift_autouse` ini gate."""
 
     def test_default_ini_true_activates(self, pytester: pytest.Pytester) -> None:
         """Plugin default (ini absent) keeps the autouse fixtures active."""
@@ -253,12 +291,12 @@ def test_inner(step):
         result.assert_outcomes(passed=1)
 
     def test_default_ini_false_skips(self, pytester: pytest.Pytester) -> None:
-        """`sift_test_results_autouse = false` makes the autouse fixtures no-op by default."""
+        """`sift_autouse = false` makes the autouse fixtures no-op by default."""
         pytester.makeconftest(_GATE_INNER_CONFTEST)
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_autouse = false
+            sift_autouse = false
             """
         )
         pytester.makepyfile(
@@ -276,7 +314,7 @@ def test_sift_include_marker_forces_on(self, pytester: pytest.Pytester) -> None:
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_autouse = false
+            sift_autouse = false
             """
         )
         pytester.makepyfile(
@@ -328,7 +366,7 @@ def test_module_pytestmark_inherits(self, pytester: pytest.Pytester) -> None:
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_autouse = false
+            sift_autouse = false
             """
         )
         pytester.makepyfile(
@@ -359,7 +397,7 @@ def test_bulk_apply_via_conftest_hook(self, pytester: pytest.Pytester) -> None:
         pytester.makepyprojecttoml(
             """
             [tool.pytest.ini_options]
-            sift_test_results_autouse = false
+            sift_autouse = false
             """
         )
         included = pytester.mkdir("included_subtree")