testserver: replace KillCaller config with HTTP kill API

Move kill-on-request behavior from test.toml fields (KillCaller,
KillCallerOffset) to a POST /__testserver/kill endpoint. Kill rules
are scoped by auth token so concurrent tests sharing a server don't
interfere.

acceptance/bin/kill_after.py is a convenience wrapper that posts to
the endpoint, keeping scripts readable.

The kill check is applied at the HTTP middleware layer (wrapping the
entire router) so it fires for all requests, including those that
would otherwise fall through to the not-found handler.

Co-authored-by: Isaac

Author: denik

acceptance/bin/kill_after.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Set up a kill rule on the testserver for the current test token.
+
+Usage: kill_after.py PATTERN OFFSET TIMES
+
+  PATTERN   HTTP method and path, e.g. "POST /api/2.2/jobs/create"
+  OFFSET    number of requests to let through before killing starts
+  TIMES     number of times to kill the caller
+
+The rule is scoped to the current DATABRICKS_TOKEN so it only affects
+the test that registers it, even when tests share a server.
+"""
+
+import json
+import os
+import sys
+import urllib.request
+
+host = os.environ.get("DATABRICKS_HOST", "")
+token = os.environ.get("DATABRICKS_TOKEN", "")
+
+if not host:
+    print("DATABRICKS_HOST not set", file=sys.stderr)
+    sys.exit(1)
+
+if len(sys.argv) != 4:
+    print(f"usage: {sys.argv[0]} PATTERN OFFSET TIMES", file=sys.stderr)
+    sys.exit(1)
+
+pattern, offset, times = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
+
+data = json.dumps({"pattern": pattern, "offset": offset, "times": times}).encode()
+req = urllib.request.Request(
+    f"{host}/__testserver/kill",
+    data=data,
+    headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
+    method="POST",
+)
+urllib.request.urlopen(req)

acceptance/bundle/deploy/wal/chain-3-jobs/script

@@ -1,3 +1,5 @@
+kill_after.py "POST /api/2.2/jobs/create" 2 1
+
 echo "=== First deploy (crashes on job_03) ==="
 trace errcode $CLI bundle deploy
 

acceptance/bundle/deploy/wal/chain-3-jobs/test.toml

@@ -3,8 +3,6 @@
 
 [[Server]]
 Pattern = "POST /api/2.2/jobs/create"
-KillCallerOffset = 2
-KillCaller = 1
 Response.Body = '{"job_id": 1001}'
 
 [[Server]]

acceptance/bundle/deploy/wal/crash-after-create/script

@@ -1,3 +1,5 @@
+kill_after.py "GET /api/2.2/jobs/get" 0 1
+
 echo "=== First deploy (crashes after job_a create, before job_b) ==="
 trace errcode $CLI bundle deploy
 

acceptance/bundle/deploy/wal/crash-after-create/test.toml

@@ -12,5 +12,4 @@ Response.Body = '{}'
 
 [[Server]]
 Pattern = "GET /api/2.2/jobs/get"
-KillCaller = 1
 Response.Body = '{"job_id": 1001, "settings": {"name": "test-job"}}'

acceptance/internal/config.go

@@ -153,18 +153,6 @@ type ServerStub struct {
 	// Configure as "1ms", "2s", "3m", etc.
 	// See [time.ParseDuration] for details.
 	Delay time.Duration
-
-	// Number of times to kill the caller process before returning normal responses.
-	// 0 = never kill (default), 1 = kill once then allow, 2 = kill twice then allow, etc.
-	// Useful for testing crash recovery scenarios where first deploy crashes but retry succeeds.
-	// Requires DATABRICKS_CLI_TEST_PID=1 to be set in the test environment.
-	KillCaller int
-
-	// Number of requests to let pass before starting to kill.
-	// Combined with KillCaller, this creates a window: requests 1 to Offset succeed,
-	// requests Offset+1 to Offset+KillCaller are killed, rest succeed.
-	// Example: KillCallerOffset=9, KillCaller=1 means let 9 requests pass, kill the 10th.
-	KillCallerOffset int
 }
 
 // FindConfigs finds all the config relevant for this test,

acceptance/internal/prepare_server.go

@@ -183,10 +183,6 @@ func startLocalServer(t *testing.T,
 		s.ResponseCallback = logResponseCallback(t)
 	}
 
-	killCounters := make(map[string]int)
-	offsetCounters := make(map[string]int)
-	killCountersMu := &sync.Mutex{}
-
 	for ind := range stubs {
 		// Later stubs take precedence over earlier ones (leaf configs override parent configs).
 		// The first handler registered for a given pattern wins, so we reverse the order.
@@ -195,11 +191,6 @@ func startLocalServer(t *testing.T,
 		items := strings.Split(stub.Pattern, " ")
 		require.Len(t, items, 2)
 
-		if stub.KillCaller > 0 {
-			killCounters[stub.Pattern] = stub.KillCaller
-			offsetCounters[stub.Pattern] = stub.KillCallerOffset
-		}
-
 		s.Handle(items[0], items[1], func(req testserver.Request) any {
 			if stub.Delay > 0 {
 				ctx := req.Context
@@ -218,10 +209,6 @@ func startLocalServer(t *testing.T,
 				}
 			}
 
-			if shouldKillCaller(stub, offsetCounters, killCounters, killCountersMu) {
-				killCaller(t, stub.Pattern, req.Headers)
-			}
-
 			return stub.Response
 		})
 	}
@@ -232,50 +219,6 @@ func startLocalServer(t *testing.T,
 	return s.URL
 }
 
-func shouldKillCaller(stub ServerStub, offsetCounters, killCounters map[string]int, mu *sync.Mutex) bool {
-	if stub.KillCaller <= 0 {
-		return false
-	}
-	mu.Lock()
-	defer mu.Unlock()
-
-	if offsetCounters[stub.Pattern] > 0 {
-		offsetCounters[stub.Pattern]--
-		return false
-	}
-
-	if killCounters[stub.Pattern] <= 0 {
-		return false
-	}
-	killCounters[stub.Pattern]--
-	return true
-}
-
-func killCaller(t *testing.T, pattern string, headers http.Header) {
-	pid := testserver.ExtractPidFromHeaders(headers)
-	if pid == 0 {
-		t.Errorf("KillCaller configured but test-pid not found in User-Agent")
-		return
-	}
-
-	process, err := os.FindProcess(pid)
-	if err != nil {
-		t.Errorf("Failed to find process %d: %s", pid, err)
-		return
-	}
-
-	// Use process.Kill() for cross-platform compatibility.
-	// On Unix, this sends SIGKILL. On Windows, this calls TerminateProcess.
-	if err := process.Kill(); err != nil {
-		t.Errorf("Failed to kill process %d: %s", pid, err)
-		return
-	}
-
-	if !waitForProcessExit(pid, 2*time.Second) {
-		t.Logf("KillCaller: timed out waiting for PID %d to exit (pattern: %s)", pid, pattern)
-	}
-	t.Logf("KillCaller: killed PID %d (pattern: %s)", pid, pattern)
-}
 
 func startProxyServer(t *testing.T,
 	recordRequests bool,

acceptance/selftest/kill_caller/currentuser/script

@@ -1,2 +1,3 @@
+kill_after.py "GET /api/2.0/preview/scim/v2/Me" 0 1
 trace errcode $CLI current-user me
 echo "Script continued after kill"

acceptance/selftest/kill_caller/currentuser/test.toml

@@ -1,4 +1 @@
 # Kill the CLI when it calls /Me endpoint (once, then allow)
-[[Server]]
-Pattern = "GET /api/2.0/preview/scim/v2/Me"
-KillCaller = 1

acceptance/selftest/kill_caller/multi_pattern/output.txt

@@ -13,8 +13,8 @@ Me attempt 2 done
 
 >>> [CLI] current-user me
 {
-  "id": "123",
-  "userName": "test@example.com"
+  "id": "[USERID]",
+  "userName": "[USERNAME]"
 }
 Me attempt 3 done - success!