repeated empirical evidence suggests that the server will time out the task when it feels like it, not on our timeline

Author: chrishagglund-ship-it

pyproject.toml

@@ -192,17 +192,15 @@ line-ending = "auto"
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 markers = [
-<<<<<<< HEAD
     "integration: requires a live Agentspan-compatible server",
     "e2e: requires a live server and AGENTSPAN_SERVER_URL",
     "sse: requires SSE streaming (AGENTSPAN_STREAMING_ENABLED=true)",
     "agent_correctness: marks tests as agent correctness tests",
     "semantic: marks tests that use an LLM judge for semantic assertions",
-=======
     "slow_sync: long-running sync lease-extension tests (~90s)",
     "slow_async: long-running async lease-extension tests (~90s)",
     "slow_test_all: long-running aggregate workflow-client test_all (~83s)",
->>>>>>> 973c79cf (wip, testing/fixes)
+    "server_timeout_unreliable: depends on server-side task timeout firing in a bounded window; excluded from CI",
 ]
 
 [tool.coverage.run]

scripts/run_integration_tests.sh

@@ -33,6 +33,11 @@
 #   test-all    aggregate workflow-client test_all (test_workflow_client_intg.py, ~83s)
 #   all         the full suite (no bucket filtering) — for a complete local run
 #
+# The long-* buckets deselect the no-heartbeat test_02 cases (marker
+# server_timeout_unreliable): those assert a server-side task timeout that the
+# sdkdev server doesn't reliably fire on a CI-bounded timeline. Use --bucket=all
+# (or target the test directly) to run them anyway.
+#
 #   ./scripts/run_integration_tests.sh                  # fast: skips slow buckets
 #   ./scripts/run_integration_tests.sh --bucket=long-sync
 #   ./scripts/run_integration_tests.sh --bucket=all     # run everything
@@ -113,13 +118,17 @@ case "$bucket" in
   # never in scope and perf_ignore is unnecessary here. This also means
   # --with-perf has no effect on these buckets; the perf test is only reachable
   # via the core/all buckets.
+  # The no-heartbeat "server should time the task out" cases (test_02) are
+  # deselected here: the sdkdev server does not reliably time a task out on a
+  # CI-bounded timeline, so they flake for reasons unrelated to the SDK. They
+  # carry the server_timeout_unreliable marker and still run under --bucket=all.
   long-sync)
     targets=("$LEASE_SYNC")
-    select=(-m slow_sync)
+    select=(-m "slow_sync and not server_timeout_unreliable")
     ;;
   long-async)
     targets=("$LEASE_ASYNC")
-    select=(-m slow_async)
+    select=(-m "slow_async and not server_timeout_unreliable")
     ;;
   test-all)
     targets=("$TEST_ALL_FILE")

tests/integration/test_async_lease_extension.py

@@ -377,6 +377,15 @@ def test_01_async_with_heartbeat_completes(self):
         self.assertEqual(task.output_data.get('slept'), TASK_SLEEP_SECONDS)
         print("\n  PASS: Async task completed with heartbeat keeping lease alive")
 
+    # NOTE: excluded from the long-* CI buckets. This scenario asserts a purely
+    # server-side mechanic — the server timing out the task's expired lease on
+    # its own. The sdkdev server does not seem to consistently/reliably time out
+    # the task by itself on a timeline that works with integration testing, so
+    # gating CI on it produces flakes unrelated to the SDK. The SDK behaviour
+    # (heartbeats keeping a task alive) is still covered by test_01.
+    # Deselected via `-m "... and not server_timeout_unreliable"`; still runs
+    # under --bucket=all or when targeted directly.
+    @pytest.mark.server_timeout_unreliable
     def test_02_async_without_heartbeat_times_out(self):
         """Async task WITHOUT lease_extend_enabled times out when sleep > responseTimeout."""
         print("\n" + "=" * 80)

tests/integration/test_lease_extension.py

@@ -318,6 +318,15 @@ def test_01_with_heartbeat_completes(self):
         finally:
             stop_workers()
 
+    # NOTE: excluded from the long-* CI buckets. This scenario asserts a purely
+    # server-side mechanic — the server timing out the task's expired lease on
+    # its own. The sdkdev server does not seem to consistently/reliably time out
+    # the task by itself on a timeline that works with integration testing, so
+    # gating CI on it produces flakes unrelated to the SDK. The SDK behaviour
+    # (heartbeats keeping a task alive) is still covered by test_01.
+    # Deselected via `-m "... and not server_timeout_unreliable"`; still runs
+    # under --bucket=all or when targeted directly.
+    @pytest.mark.server_timeout_unreliable
     def test_02_without_heartbeat_times_out(self):
         """Task WITHOUT lease_extend_enabled times out when sleep > responseTimeout."""
         print("\n" + "=" * 80)