fix: api-proxy auth chain — trim keys, align placeholder format, add diagnostics (#1528)

* fix: api-proxy auth chain — trim keys, align placeholder format, add diagnostics

Three fixes for the Anthropic API proxy auth chain failure (#1527):

1. Trim API keys read from env vars in api-proxy (server.js)
   CI secrets and docker-compose YAML may include trailing whitespace
   or newlines. Untrimmed keys produce malformed HTTP headers (the
   newline terminates the header prematurely), causing silent auth
   failures with error_status: null.

2. Align ANTHROPIC_AUTH_TOKEN placeholder with sk-ant- key format
   Claude Code validates key format before making API calls. The old
   placeholder 'placeholder-token-for-credential-isolation' lacks the
   sk-ant- prefix and may fail format validation. Now uses the same
   value as apiKeyHelper: 'sk-ant-placeholder-key-for-credential-isolation'.

3. Add auth header injection + upstream error logging in api-proxy
   - 'auth_inject' debug log confirms key injection (length, preview)
   - 'upstream_auth_error' warning on 401/403 from upstream API

Closes #1527

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: update health check to match new ANTHROPIC_AUTH_TOKEN placeholder

The api-proxy-health-check.sh validates that ANTHROPIC_AUTH_TOKEN
contains a placeholder value. Update the expected value to match
the new sk-ant- prefixed placeholder set in docker-manager.ts.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: lpcox

containers/agent/api-proxy-health-check.sh

@@ -33,9 +33,9 @@ if [ -n "$ANTHROPIC_BASE_URL" ]; then
 
   # Verify ANTHROPIC_AUTH_TOKEN is placeholder (if present)
   if [ -n "$ANTHROPIC_AUTH_TOKEN" ]; then
-    if [ "$ANTHROPIC_AUTH_TOKEN" != "placeholder-token-for-credential-isolation" ]; then
+    if [ "$ANTHROPIC_AUTH_TOKEN" != "sk-ant-placeholder-key-for-credential-isolation" ]; then
       echo "[health-check][ERROR] ANTHROPIC_AUTH_TOKEN contains non-placeholder value!"
-      echo "[health-check][ERROR] Token should be 'placeholder-token-for-credential-isolation'"
+      echo "[health-check][ERROR] Token should be 'sk-ant-placeholder-key-for-credential-isolation'"
       exit 1
     fi
     echo "[health-check] ✓ ANTHROPIC_AUTH_TOKEN is placeholder value (correct)"

containers/api-proxy/server.js

@@ -43,9 +43,11 @@ function shouldStripHeader(name) {
 }
 
 // Read API keys from environment (set by docker-compose)
-const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
-const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
-const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN;
+// Trim whitespace/newlines to prevent malformed HTTP headers — env vars from
+// CI secrets or docker-compose YAML may include trailing whitespace.
+const OPENAI_API_KEY = (process.env.OPENAI_API_KEY || '').trim() || undefined;
+const ANTHROPIC_API_KEY = (process.env.ANTHROPIC_API_KEY || '').trim() || undefined;
+const COPILOT_GITHUB_TOKEN = (process.env.COPILOT_GITHUB_TOKEN || '').trim() || undefined;
 
 // Configurable API target hosts (supports custom endpoints / internal LLM routers)
 const OPENAI_API_TARGET = process.env.OPENAI_API_TARGET || 'api.openai.com';
@@ -332,6 +334,21 @@ function proxyRequest(req, res, targetHost, injectHeaders, provider, basePath =
     headers['x-request-id'] = requestId;
     Object.assign(headers, injectHeaders);
 
+    // Log auth header injection for debugging credential-isolation issues
+    const injectedKey = injectHeaders['x-api-key'] || injectHeaders['authorization'];
+    if (injectedKey) {
+      const keyPreview = injectedKey.length > 8
+        ? `${injectedKey.substring(0, 8)}...${injectedKey.substring(injectedKey.length - 4)}`
+        : '(short)';
+      logRequest('debug', 'auth_inject', {
+        request_id: requestId,
+        provider,
+        key_length: injectedKey.length,
+        key_preview: keyPreview,
+        has_anthropic_version: !!headers['anthropic-version'],
+      });
+    }
+
     const options = {
       hostname: targetHost,
       port: 443,
@@ -391,6 +408,19 @@ function proxyRequest(req, res, targetHost, injectHeaders, provider, basePath =
 
       // Copy response headers and add X-Request-ID
       const resHeaders = { ...proxyRes.headers, 'x-request-id': requestId };
+
+      // Log upstream auth failures prominently for debugging
+      if (proxyRes.statusCode === 401 || proxyRes.statusCode === 403) {
+        logRequest('warn', 'upstream_auth_error', {
+          request_id: requestId,
+          provider,
+          status: proxyRes.statusCode,
+          upstream_host: targetHost,
+          path: sanitizeForLog(req.url),
+          message: `Upstream returned ${proxyRes.statusCode} — check that the API key is valid and has not expired`,
+        });
+      }
+
       res.writeHead(proxyRes.statusCode, resHeaders);
       proxyRes.pipe(res);
     });

src/docker-manager.test.ts

@@ -2048,7 +2048,7 @@ describe('docker-manager', () => {
         const agent = result.services.agent;
         const env = agent.environment as Record<string, string>;
         expect(env.ANTHROPIC_BASE_URL).toBe('http://172.30.0.30:10001');
-        expect(env.ANTHROPIC_AUTH_TOKEN).toBe('placeholder-token-for-credential-isolation');
+        expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-ant-placeholder-key-for-credential-isolation');
         expect(env.CLAUDE_CODE_API_KEY_HELPER).toBe('/usr/local/bin/get-claude-key.sh');
       });
 
@@ -2059,7 +2059,7 @@ describe('docker-manager', () => {
         const env = agent.environment as Record<string, string>;
         expect(env.OPENAI_BASE_URL).toBe('http://172.30.0.30:10000/v1');
         expect(env.ANTHROPIC_BASE_URL).toBe('http://172.30.0.30:10001');
-        expect(env.ANTHROPIC_AUTH_TOKEN).toBe('placeholder-token-for-credential-isolation');
+        expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-ant-placeholder-key-for-credential-isolation');
         expect(env.CLAUDE_CODE_API_KEY_HELPER).toBe('/usr/local/bin/get-claude-key.sh');
       });
 
@@ -2070,7 +2070,7 @@ describe('docker-manager', () => {
         const env = agent.environment as Record<string, string>;
         expect(env.OPENAI_BASE_URL).toBeUndefined();
         expect(env.ANTHROPIC_BASE_URL).toBe('http://172.30.0.30:10001');
-        expect(env.ANTHROPIC_AUTH_TOKEN).toBe('placeholder-token-for-credential-isolation');
+        expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-ant-placeholder-key-for-credential-isolation');
         expect(env.CLAUDE_CODE_API_KEY_HELPER).toBe('/usr/local/bin/get-claude-key.sh');
       });
 
@@ -2130,7 +2130,7 @@ describe('docker-manager', () => {
           // Agent should have the BASE_URL to reach the sidecar instead
           expect(env.ANTHROPIC_BASE_URL).toBe('http://172.30.0.30:10001');
           // Agent should have placeholder token for Claude Code compatibility
-          expect(env.ANTHROPIC_AUTH_TOKEN).toBe('placeholder-token-for-credential-isolation');
+          expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-ant-placeholder-key-for-credential-isolation');
         } finally {
           if (origKey !== undefined) {
             process.env.ANTHROPIC_API_KEY = origKey;
@@ -2219,7 +2219,7 @@ describe('docker-manager', () => {
           expect(env.ANTHROPIC_API_KEY).toBeUndefined();
           expect(env.ANTHROPIC_BASE_URL).toBe('http://172.30.0.30:10001');
           // But should have placeholder token for Claude Code compatibility
-          expect(env.ANTHROPIC_AUTH_TOKEN).toBe('placeholder-token-for-credential-isolation');
+          expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-ant-placeholder-key-for-credential-isolation');
         } finally {
           if (origKey !== undefined) {
             process.env.ANTHROPIC_API_KEY = origKey;

src/docker-manager.ts

@@ -1447,7 +1447,8 @@ export function generateDockerCompose(
 
       // Set placeholder token for Claude Code CLI compatibility
       // Real authentication happens via ANTHROPIC_BASE_URL pointing to api-proxy
-      environment.ANTHROPIC_AUTH_TOKEN = 'placeholder-token-for-credential-isolation';
+      // Use sk-ant- prefix so Claude Code's key-format validation passes
+      environment.ANTHROPIC_AUTH_TOKEN = 'sk-ant-placeholder-key-for-credential-isolation';
       logger.debug('ANTHROPIC_AUTH_TOKEN set to placeholder value for credential isolation');
 
       // Set API key helper for Claude Code CLI to use credential isolation

tests/integration/api-proxy.test.ts

@@ -101,7 +101,7 @@ describe('API Proxy Sidecar', () => {
     );
 
     expect(result).toSucceed();
-    expect(result.stdout).toContain('ANTHROPIC_AUTH_TOKEN=placeholder-token-for-credential-isolation');
+    expect(result.stdout).toContain('ANTHROPIC_AUTH_TOKEN=sk-ant-placeholder-key-for-credential-isolation');
   }, 180000);
 
   test('should set OPENAI_BASE_URL in agent when OpenAI key is provided', async () => {