feat: auto failover for API with LK Cloud (#936)

* feat: auto failover for API with LK Cloud

automatically fail over 5xx and transport errors

* fix codeql perms

* tweaks

* address feedback

* clean up failover logic, handle per-attempt timeout

* sane default timeouts for CreateSIPParticipant

Author: davidzhao

.github/workflows/test-api.yml

@@ -0,0 +1,60 @@
+# Copyright 2026 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Test API
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  failover:
+    runs-on: ubuntu-latest
+    services:
+      mock-server:
+        image: livekit/test-server:latest
+        ports:
+          - 9999:9999
+          - 10000:10000
+          - 10001:10001
+          - 10002:10002
+    steps:
+      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6
+
+      - name: Set up Opus
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsoxr-dev libopus-dev libopusfile-dev
+
+      - name: Set up Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+        with:
+          go-version: 1.26.0
+
+      - name: Wait for mock server
+        run: |
+          for i in $(seq 1 30); do
+            curl -sf http://127.0.0.1:9999/settings/regions >/dev/null && exit 0
+            sleep 1
+          done
+          echo "mock server did not become ready" && exit 1
+
+      - name: Run API tests
+        run: go test -v -count=1 -run '^TestAPI_' .

agent_client.go

@@ -29,7 +29,7 @@ func NewAgentClient(url string, apiKey string, apiSecret string, opts ...AgentCl
 	}
 	c := &AgentClient{
 		authBase:   authBase{apiKey, apiSecret},
-		httpClient: &http.Client{},
+		httpClient: newAPIHTTPClient(),
 	}
 	for _, opt := range opts {
 		opt(c)
@@ -53,143 +53,143 @@ func WithTwirpClientOptions(opts ...twirp.ClientOption) AgentClientOption {
 }
 
 func (c *AgentClient) CreateAgent(ctx context.Context, req *livekit.CreateAgentRequest) (*livekit.CreateAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.CreateAgent(ctx, req)
 }
 
 func (c *AgentClient) CreateAgentV2(ctx context.Context, req *livekit.CreateAgentV2Request) (*livekit.CreateAgentV2Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.CreateAgentV2(ctx, req)
 }
 
 func (c *AgentClient) ListAgents(ctx context.Context, req *livekit.ListAgentsRequest) (*livekit.ListAgentsResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.ListAgents(ctx, req)
 }
 
 func (c *AgentClient) ListAgentVersions(ctx context.Context, req *livekit.ListAgentVersionsRequest) (*livekit.ListAgentVersionsResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.ListAgentVersions(ctx, req)
 }
 
 func (c *AgentClient) DeleteAgent(ctx context.Context, req *livekit.DeleteAgentRequest) (*livekit.DeleteAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.DeleteAgent(ctx, req)
 }
 
 func (c *AgentClient) UpdateAgent(ctx context.Context, req *livekit.UpdateAgentRequest) (*livekit.UpdateAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.UpdateAgent(ctx, req)
 }
 
 func (c *AgentClient) RestartAgent(ctx context.Context, req *livekit.RestartAgentRequest) (*livekit.RestartAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.RestartAgent(ctx, req)
 }
 
 func (c *AgentClient) RollbackAgent(ctx context.Context, req *livekit.RollbackAgentRequest) (*livekit.RollbackAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.RollbackAgent(ctx, req)
 }
 
 func (c *AgentClient) ListAgentSecrets(ctx context.Context, req *livekit.ListAgentSecretsRequest) (*livekit.ListAgentSecretsResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.ListAgentSecrets(ctx, req)
 }
 
 func (c *AgentClient) UpdateAgentSecrets(ctx context.Context, req *livekit.UpdateAgentSecretsRequest) (*livekit.UpdateAgentSecretsResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.UpdateAgentSecrets(ctx, req)
 }
 
 func (c *AgentClient) DeployAgent(ctx context.Context, req *livekit.DeployAgentRequest) (*livekit.DeployAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.DeployAgent(ctx, req)
 }
 
 func (c *AgentClient) DeployAgentV2(ctx context.Context, req *livekit.DeployAgentV2Request) (*livekit.DeployAgentV2Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.DeployAgentV2(ctx, req)
 }
 
 func (c *AgentClient) PromoteAgent(ctx context.Context, req *livekit.PromoteAgentRequest) (*livekit.PromoteAgentResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.PromoteAgent(ctx, req)
 }
 
 func (c *AgentClient) GetClientSettings(ctx context.Context, req *livekit.ClientSettingsRequest) (*livekit.ClientSettingsResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.GetClientSettings(ctx, req)
 }
 
 func (c *AgentClient) CreatePrivateLink(ctx context.Context, req *livekit.CreatePrivateLinkRequest) (*livekit.CreatePrivateLinkResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.CreatePrivateLink(ctx, req)
 }
 
 func (c *AgentClient) DestroyPrivateLink(ctx context.Context, req *livekit.DestroyPrivateLinkRequest) (*livekit.DestroyPrivateLinkResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.DestroyPrivateLink(ctx, req)
 }
 
 func (c *AgentClient) ListPrivateLinks(ctx context.Context, req *livekit.ListPrivateLinksRequest) (*livekit.ListPrivateLinksResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.agentClient.ListPrivateLinks(ctx, req)
 }
 
 func (c *AgentClient) GetPrivateLinkStatus(ctx context.Context, req *livekit.GetPrivateLinkStatusRequest) (*livekit.GetPrivateLinkStatusResponse, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{Admin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{Admin: true})
 	if err != nil {
 		return nil, err
 	}

agent_dispatch_client.go

@@ -16,7 +16,6 @@ package lksdk
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/twitchtv/twirp"
 
@@ -31,7 +30,7 @@ type AgentDispatchClient struct {
 
 func NewAgentDispatchServiceClient(url string, apiKey string, secretKey string, opts ...twirp.ClientOption) *AgentDispatchClient {
 	url = signalling.ToHttpURL(url)
-	client := livekit.NewAgentDispatchServiceProtobufClient(url, &http.Client{}, opts...)
+	client := livekit.NewAgentDispatchServiceProtobufClient(url, newAPIHTTPClient(), opts...)
 
 	return &AgentDispatchClient{
 		agentDispatchService: client,
@@ -43,7 +42,7 @@ func NewAgentDispatchServiceClient(url string, apiKey string, secretKey string,
 }
 
 func (c *AgentDispatchClient) CreateDispatch(ctx context.Context, req *livekit.CreateAgentDispatchRequest) (*livekit.AgentDispatch, error) {
-	ctx, err := c.withAuth(ctx, withVideoGrant{RoomAdmin: true, Room: req.Room})
+	ctx, err := c.prepareContext(ctx, withVideoGrant{RoomAdmin: true, Room: req.Room})
 	if err != nil {
 		return nil, err
 	}
@@ -52,7 +51,7 @@ func (c *AgentDispatchClient) CreateDispatch(ctx context.Context, req *livekit.C
 }
 
 func (c *AgentDispatchClient) DeleteDispatch(ctx context.Context, req *livekit.DeleteAgentDispatchRequest) (*livekit.AgentDispatch, error) {
-	ctx, err := c.withAuth(ctx, withVideoGrant{RoomAdmin: true, Room: req.Room})
+	ctx, err := c.prepareContext(ctx, withVideoGrant{RoomAdmin: true, Room: req.Room})
 	if err != nil {
 		return nil, err
 	}
@@ -61,7 +60,7 @@ func (c *AgentDispatchClient) DeleteDispatch(ctx context.Context, req *livekit.D
 }
 
 func (c *AgentDispatchClient) ListDispatch(ctx context.Context, req *livekit.ListAgentDispatchRequest) (*livekit.ListAgentDispatchResponse, error) {
-	ctx, err := c.withAuth(ctx, withVideoGrant{RoomAdmin: true, Room: req.Room})
+	ctx, err := c.prepareContext(ctx, withVideoGrant{RoomAdmin: true, Room: req.Room})
 	if err != nil {
 		return nil, err
 	}

agent_simulation_client.go

@@ -16,7 +16,6 @@ package lksdk
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/twitchtv/twirp"
 
@@ -29,55 +28,55 @@ type AgentSimulationClient struct {
 }
 
 func NewAgentSimulationClient(url string, apiKey string, apiSecret string, opts ...twirp.ClientOption) *AgentSimulationClient {
-	client := livekit.NewAgentSimulationProtobufClient(url, &http.Client{}, opts...)
+	client := livekit.NewAgentSimulationProtobufClient(url, newAPIHTTPClient(), opts...)
 	return &AgentSimulationClient{
 		simulationClient: client,
 		authBase:         authBase{apiKey, apiSecret},
 	}
 }
 
 func (c *AgentSimulationClient) CreateSimulationRun(ctx context.Context, req *livekit.SimulationRun_Create_Request) (*livekit.SimulationRun_Create_Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{SimulationAdmin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{SimulationAdmin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.simulationClient.CreateSimulationRun(ctx, req)
 }
 
 func (c *AgentSimulationClient) ConfirmSimulationSourceUpload(ctx context.Context, req *livekit.SimulationRun_ConfirmSourceUpload_Request) (*livekit.SimulationRun_ConfirmSourceUpload_Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{SimulationAdmin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{SimulationAdmin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.simulationClient.ConfirmSimulationSourceUpload(ctx, req)
 }
 
 func (c *AgentSimulationClient) GetSimulationRun(ctx context.Context, req *livekit.SimulationRun_Get_Request) (*livekit.SimulationRun_Get_Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{SimulationAdmin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{SimulationAdmin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.simulationClient.GetSimulationRun(ctx, req)
 }
 
 func (c *AgentSimulationClient) ListSimulationRuns(ctx context.Context, req *livekit.SimulationRun_List_Request) (*livekit.SimulationRun_List_Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{SimulationAdmin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{SimulationAdmin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.simulationClient.ListSimulationRuns(ctx, req)
 }
 
 func (c *AgentSimulationClient) CancelSimulationRun(ctx context.Context, req *livekit.SimulationRun_Cancel_Request) (*livekit.SimulationRun_Cancel_Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{SimulationAdmin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{SimulationAdmin: true})
 	if err != nil {
 		return nil, err
 	}
 	return c.simulationClient.CancelSimulationRun(ctx, req)
 }
 
 func (c *AgentSimulationClient) CreateScenarioFromSession(ctx context.Context, req *livekit.Scenario_CreateFromSession_Request) (*livekit.Scenario_CreateFromSession_Response, error) {
-	ctx, err := c.withAuth(ctx, withAgentGrant{SimulationAdmin: true})
+	ctx, err := c.prepareContext(ctx, withAgentGrant{SimulationAdmin: true})
 	if err != nil {
 		return nil, err
 	}

auth.go

@@ -52,7 +52,11 @@ func (g withAgentGrant) Apply(t *auth.AccessToken) {
 	t.SetAgentGrant((*auth.AgentGrant)(&g))
 }
 
-func (b authBase) withAuth(ctx context.Context, opt authOption, options ...authOption) (context.Context, error) {
+// prepareContext builds the context for an outgoing API request: it signs an
+// access token for the given grants and attaches it as a request header, then
+// detaches a long-enough deadline so failover can reset it per attempt (see
+// withFailoverTimeout).
+func (b authBase) prepareContext(ctx context.Context, opt authOption, options ...authOption) (context.Context, error) {
 	at := auth.NewAccessToken(b.apiKey, b.apiSecret)
 	opt.Apply(at)
 	for _, opt := range options {
@@ -78,5 +82,10 @@ func (b authBase) withAuth(ctx context.Context, opt authOption, options ...authO
 		}
 	}
 
+	// Detach a long-enough deadline so it isn't enforced across failover retries
+	// (twirp re-checks the context after each request); the transport re-applies
+	// the budget per attempt. A no-op for short or deadline-free requests.
+	ctx = withFailoverTimeout(ctx)
+
 	return twirp.WithHTTPRequestHeaders(ctx, ctxH)
 }