fix(test): defeat Linear API's transient "conflict on insert" flake

flipbit03 · flipbit03 · commit 5d23aa8450dc · 2026-04-17T11:03:09.000-03:00
CI's Tests (Online) job has been flaking on `projects create` for several PRs. Linear's API has a transient failure mode where `*Create` mutations return "Entity X with id <UUID> already exists" for a UUID that doesn't actually exist server-side (verified by `read` immediately returning "Entity not found"). The same pattern hit PR #140's post-merge CI before the work in #141 even started. Local probing shows a clear cold-start signature: from a fresh process, the first ~3 attempts at `projects create` fail back-to-back, then it abruptly starts working (next 7 succeed in a row). The previous helper's 3-retry, 14s-backoff budget routinely runs out before the cold window closes. Layered fix: 1. **`run_lineark_with_retry`** (in tests/online.rs): - Up to 8 attempts with `[0, 2, 5, 10, 20, 30, 45, 60]`s backoff (~172s worst case) so we ride out the cold window. - Mutate the request body each retry by appending a fresh suffix to the `<name>` positional. Linear's stuck UUIDs are keyed on body content, so a different body avoids the cached state. - Returns `(Output, String)` where the second element is the name actually used. Callers shadow their `unique_name` with this so downstream `read by name` lookups match server state. 2. **CI workflow + Makefile** (`ci.yml`, `ci-online-fork.yml`, `Makefile`): wrap the whole online suite in a 3x retry with a workspace clean between attempts. Belt-and-suspenders for the occasional case where even the per-call helper's budget isn't enough.
diff --git a/.github/workflows/ci-online-fork.yml b/.github/workflows/ci-online-fork.yml
@@ -52,7 +52,21 @@ jobs:
         run: cargo run -p lineark-test-utils --bin cleanup-test-workspace
 
       - name: Run online tests
-        run: cargo test --workspace --test online -- --test-threads=1
+        # See ci.yml comment — Linear's API has a transient cold-start
+        # failure on `*Create` mutations; wrap in 3x retry to absorb it.
+        run: |
+          for attempt in 1 2 3; do
+            echo ">>> online attempt $attempt/3"
+            if cargo test --workspace --test online -- --test-threads=1; then
+              echo ">>> online passed on attempt $attempt"
+              exit 0
+            fi
+            echo ">>> online attempt $attempt failed; cleaning workspace before retry"
+            cargo run -p lineark-test-utils --bin cleanup-test-workspace || true
+            sleep 30
+          done
+          echo ">>> online failed after 3 attempts"
+          exit 1
 
       - name: Clean test workspace (post)
         if: always()
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -107,7 +107,25 @@ jobs:
         run: cargo run -p lineark-test-utils --bin cleanup-test-workspace
 
       - name: Run online tests
-        run: cargo test --workspace --test online -- --test-threads=1
+        # Linear's API has a known transient failure mode on `*Create`
+        # mutations (returns "conflict on insert" with a phantom UUID — see
+        # the helper docs in crates/lineark/tests/online.rs). The per-call
+        # retry there handles most cases, but the cold-start window is
+        # occasionally longer than its budget. Wrap the whole suite in a 3x
+        # retry with a workspace clean between attempts.
+        run: |
+          for attempt in 1 2 3; do
+            echo ">>> online attempt $attempt/3"
+            if cargo test --workspace --test online -- --test-threads=1; then
+              echo ">>> online passed on attempt $attempt"
+              exit 0
+            fi
+            echo ">>> online attempt $attempt failed; cleaning workspace before retry"
+            cargo run -p lineark-test-utils --bin cleanup-test-workspace || true
+            sleep 30
+          done
+          echo ">>> online failed after 3 attempts"
+          exit 1
 
       - name: Clean test workspace (post)
         if: always()
diff --git a/Makefile b/Makefile
@@ -26,6 +26,22 @@ test:
 # Run online tests against the live Linear API. Requires ~/.linear_api_token_test.
 # Cleans the test workspace before running to avoid stale resource conflicts.
 # test_with's custom harness runs tests sequentially and aborts on first panic.
+#
+# Linear's API has a known transient failure mode on `*Create` mutations
+# (returns "conflict on insert" with a UUID it just generated, with no
+# matching record server-side — confirmed by `read` returning "not found").
+# The per-call helper in tests/online.rs retries with body mutation, but the
+# cold-start window is sometimes longer than the 8-attempt budget there.
+# Wrap the whole suite in a 3x retry so a single unlucky test doesn't sink CI.
 test-online:
 	cargo run -p lineark-test-utils --bin cleanup-test-workspace
-	cargo test --workspace --test online -- --test-threads=1
+	@for attempt in 1 2 3; do \
+		echo ">>> test-online attempt $$attempt/3"; \
+		if cargo test --workspace --test online -- --test-threads=1; then \
+			echo ">>> test-online passed on attempt $$attempt"; exit 0; \
+		fi; \
+		echo ">>> test-online attempt $$attempt failed; cleaning up before retry"; \
+		cargo run -p lineark-test-utils --bin cleanup-test-workspace; \
+		sleep 30; \
+	done; \
+	echo ">>> test-online failed after 3 attempts"; exit 1
diff --git a/crates/lineark/tests/online.rs b/crates/lineark/tests/online.rs
@@ -16,30 +16,78 @@ fn lineark() -> Command {
 }
 
 /// Run a lineark CLI command with retry logic.
-/// Retries up to 3 times with backoff for transient API errors (e.g., "conflict on insert").
-fn run_lineark_with_retry(args: &[&str]) -> std::process::Output {
-    for attempt in 0..3u32 {
-        if attempt > 0 {
-            std::thread::sleep(std::time::Duration::from_secs(1u64 << attempt));
-        }
-        let output = lineark()
-            .args(args)
-            .output()
-            .expect("failed to execute lineark");
-        if output.status.success() {
-            return output;
+/// Run `lineark <args>` with retry on Linear's "conflict on insert" failure.
+///
+/// The conflict is a transient Linear API bug: `*Create` returns
+/// "Entity X with id <UUID> already exists" for a UUID that doesn't
+/// actually exist server-side (verified by `read` immediately returning
+/// "Entity not found"). Empirically the API has a "cold start" window —
+/// the first few project creates from a fresh test process all conflict,
+/// then it abruptly starts working. Probes locally show 3 back-to-back
+/// failures followed by 7 successes in a row.
+///
+/// The fix:
+/// 1. **Retry persistently** — up to 8 attempts with `[0, 2, 5, 10, 20,
+///    30, 45, 60]` second backoffs (~172 s worst case) so we ride out
+///    the cold window.
+/// 2. **Mutate the request body each retry** by appending a fresh suffix
+///    to the `<name>` positional. Linear's stuck UUIDs are keyed on body
+///    content, so a different body avoids the cached state on retry.
+///
+/// Returns a tuple of `(process output, name actually used on the final
+/// attempt)`. The returned name differs from the original when retries
+/// had to mutate the body — callers that look the entity up later by
+/// name **must** use this value, not their original `unique_name`, or
+/// they'll drift from server state.
+fn run_lineark_with_retry(args: &[&str]) -> (std::process::Output, String) {
+    // Locate the `<resource> create <name>` triple once so each retry can
+    // swap a fresh suffix into the same arg slot.
+    let name_idx = args.iter().position(|&a| a == "create").map(|p| p + 1);
+    let original_name = name_idx
+        .and_then(|i| args.get(i).copied())
+        .unwrap_or("")
+        .to_string();
+
+    let mut owned: Vec<String> = args.iter().map(|s| s.to_string()).collect();
+    let mut current_name = original_name.clone();
+
+    const BACKOFFS: &[u64] = &[0, 2, 5, 10, 20, 30, 45, 60];
+
+    let mut last_output = lineark()
+        .args(args)
+        .output()
+        .expect("failed to execute lineark");
+
+    for &wait in BACKOFFS.iter().skip(1) {
+        if last_output.status.success() {
+            return (last_output, current_name);
         }
-        let stderr = String::from_utf8_lossy(&output.stderr);
-        // Only retry on transient "conflict on insert" errors from the Linear API.
+        let stderr = String::from_utf8_lossy(&last_output.stderr);
         if !stderr.contains("conflict on insert") {
-            return output;
+            return (last_output, current_name);
+        }
+
+        std::thread::sleep(std::time::Duration::from_secs(wait));
+
+        // Mutate the name with a fresh suffix so the next request body
+        // differs from the previous one — Linear's stuck UUID is keyed on
+        // body content.
+        if let Some(idx) = name_idx {
+            current_name = format!(
+                "{original_name} retry-{}",
+                &uuid::Uuid::new_v4().to_string()[..6]
+            );
+            owned[idx] = current_name.clone();
         }
+
+        let refs: Vec<&str> = owned.iter().map(String::as_str).collect();
+        last_output = lineark()
+            .args(&refs)
+            .output()
+            .expect("failed to execute lineark");
     }
-    // Final attempt without retry.
-    lineark()
-        .args(args)
-        .output()
-        .expect("failed to execute lineark")
+
+    (last_output, current_name)
 }
 
 /// Helper: create a fresh test team via the SDK.
@@ -793,7 +841,7 @@ mod online {
         let team_key = team.key.clone();
 
         // Create with --priority urgent (textual).
-        let output = run_lineark_with_retry(&[
+        let (output, _) = run_lineark_with_retry(&[
             "--api-token",
             &token,
             "--format",
@@ -2449,7 +2497,7 @@ mod online {
         let team_key = team.key.clone();
 
         // Create a project via CLI (with retry for transient "conflict on insert" errors).
-        let output = run_lineark_with_retry(&[
+        let (output, _) = run_lineark_with_retry(&[
             "--api-token",
             &token,
             "--format",
@@ -2600,7 +2648,9 @@ mod online {
         let team_key = team.key.clone();
 
         // Create a project with --lead me (with retry for transient API errors).
-        let output = run_lineark_with_retry(&[
+        // Shadow `unique_name` so subsequent reads/asserts use the name actually
+        // sent to Linear — the helper may have appended a retry suffix.
+        let (output, unique_name) = run_lineark_with_retry(&[
             "--api-token",
             &token,
             "--format",
@@ -2718,7 +2768,7 @@ mod online {
         let team_key = team.key.clone();
 
         // Create a project with --lead me --members me (with retry for transient API errors).
-        let output = run_lineark_with_retry(&[
+        let (output, _) = run_lineark_with_retry(&[
             "--api-token",
             &token,
             "--format",
@@ -2815,7 +2865,7 @@ mod online {
         let team_key = team.key.clone();
 
         // Create a project with a lead and target date set.
-        let output = run_lineark_with_retry(&[
+        let (output, _) = run_lineark_with_retry(&[
             "--api-token",
             &token,
             "--format",
@@ -4312,7 +4362,7 @@ mod online {
             "[test] CLI project filter {}",
             &uuid::Uuid::new_v4().to_string()[..8]
         );
-        let output = run_lineark_with_retry(&[
+        let (output, _) = run_lineark_with_retry(&[
             "--api-token",
             &token,
             "--format",
