fix(auth): add automatic key reload for state directory recovery (#3374)

Closes #3367. Adds reload() with mtime-based detection, auto-reload on validation failure, and isHealthy() for degraded state detection.

Author: OneStepAt4time

src/__tests__/fix-3367-auth-recovery.test.ts

@@ -0,0 +1,105 @@
+/**
+ * Issue #3367: Server can enter unrecoverable auth state where all tokens are invalid.
+ *
+ * After state directory churn (delete + ag init --force), the server should
+ * automatically recover by re-reading keys.json from disk on the next
+ * validation failure.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdir, writeFile, rm } from 'node:fs/promises';
+import { statSync } from 'node:fs';
+import { join } from 'node:path';
+import { AuthManager } from '../services/auth/AuthManager.js';
+
+describe('Issue #3367: Auth state recovery after state directory loss', () => {
+  let tmpDir: string;
+  let keysFile: string;
+  let auth: AuthManager;
+
+  beforeEach(async () => {
+    tmpDir = join('/tmp', `test-3367-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    await mkdir(tmpDir, { recursive: true });
+    keysFile = join(tmpDir, 'keys.json');
+    auth = new AuthManager(keysFile, 'master-secret');
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  it('reload() picks up new keys after state dir wipe and recreate', async () => {
+    // Step 1: Create initial key
+    const { key: oldKey } = await auth.createKey('old-key', 100, undefined, 'admin');
+    expect(auth.validate(oldKey).valid).toBe(true);
+
+    // Step 2: Delete keys.json (simulate state dir wipe)
+    await rm(keysFile);
+
+    // Step 3: Old key should still work (in-memory)
+    expect(auth.validate(oldKey).valid).toBe(true);
+
+    // Step 4: Write new keys.json (simulate ag init --force creating new token)
+    const newAuth = new AuthManager(keysFile, 'master-secret');
+    const { key: newKey } = await newAuth.createKey('new-key', 100, undefined, 'admin');
+    await newAuth.save();
+
+    // Step 5: Old key still works (in-memory), new key doesn't
+    expect(auth.validate(oldKey).valid).toBe(true);
+    expect(auth.validate(newKey).valid).toBe(false);
+
+    // Step 6: Reload should pick up new keys
+    const reloaded = await auth.reload();
+    expect(reloaded).toBe(true);
+
+    // Step 7: New key should now work, old key should not
+    expect(auth.validate(newKey).valid).toBe(true);
+    expect(auth.validate(oldKey).valid).toBe(false);
+  });
+
+  it('reload() does nothing if keys.json has not changed', async () => {
+    await auth.createKey('test-key', 100, undefined, 'admin');
+    await auth.load(); // ensure mtime is tracked
+    const reloaded = await auth.reload();
+    expect(reloaded).toBe(false);
+  });
+
+  it('reload() handles missing keys.json gracefully', async () => {
+    const { key: existingKey } = await auth.createKey('graceful-key', 100, undefined, 'admin');
+    expect(auth.validate(existingKey).valid).toBe(true);
+    await rm(keysFile);
+    const reloaded = await auth.reload();
+    expect(reloaded).toBe(false);
+    expect(auth.validate(existingKey).valid).toBe(true);
+    expect(auth.isHealthy()).toBe(false);
+  });
+
+  it('reload() is safe to call concurrently', async () => {
+    await auth.createKey('test-key', 100, undefined, 'admin');
+    const results = await Promise.all([auth.reload(), auth.reload(), auth.reload()]);
+    expect(results.every(r => typeof r === 'boolean')).toBe(true);
+  });
+
+  it('isHealthy() returns false when keys.json missing but keys exist in memory', async () => {
+    await auth.createKey('test-key', 100, undefined, 'admin');
+    expect(auth.isHealthy()).toBe(true);
+    await rm(keysFile);
+    expect(auth.isHealthy()).toBe(false);
+  });
+
+  it('isHealthy() returns true when keys.json exists', async () => {
+    await auth.createKey('test-key', 100, undefined, 'admin');
+    expect(auth.isHealthy()).toBe(true);
+  });
+
+  it('isHealthy() returns true when no keys and no file (fresh start)', () => {
+    expect(auth.isHealthy()).toBe(true);
+  });
+
+  it('save() updates mtime so subsequent reload() is a no-op', async () => {
+    await auth.createKey('key1', 100, undefined, 'admin');
+    const mtimeAfterSave = statSync(keysFile).mtimeMs;
+    const reloaded = await auth.reload();
+    expect(reloaded).toBe(false);
+  });
+});

src/server.ts

@@ -902,7 +902,7 @@ async function main(): Promise<void> {
       await auth.load();
     },
     stop: async () => {},
-    health: async () => ({ healthy: true }),
+    health: async () => ({ healthy: auth.isHealthy(), details: auth.isHealthy() ? undefined : "keys.json missing — state dir may have been wiped" }),
   });
   container.register('channelManager', channels, {
     start: async () => {

src/services/auth/AuthManager.ts

@@ -10,7 +10,7 @@ import { createHash, randomBytes } from 'node:crypto';
 import { timingSafeStringEqual } from '../../crypto-utils.js';
 import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
 import { authStoreSchema } from '../../validation.js';
-import { existsSync } from 'node:fs';
+import { existsSync, statSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { secureFilePermissions } from '../../file-utils.js';
 import { SYSTEM_TENANT } from '../../config.js';
@@ -86,6 +86,10 @@ export class AuthManager {
   private audit: AuditLogger | null = null;
   /** #2534: Dirty flag — set when lastUsedAt is updated, cleared when persisted to disk. */
   private lastUsedAtDirty = false;
+  /** #3367: Track mtime of keys.json to detect external changes. */
+  private lastKeysMtime: number | null = null;
+  /** #3367: Guard against concurrent reloads. */
+  private reloading = false;
 
 
   constructor(
@@ -136,6 +140,9 @@ export class AuthManager {
             await this.save();
           }
         }
+        // #3367: Track mtime after load
+        try { this.lastKeysMtime = statSync(this.keysFile).mtimeMs; } catch { /* ignore */ }
+
         // Issue #2097: Load grace keys from persisted store
         if (Array.isArray(parsed.graceKeys)) {
           const now = Date.now();
@@ -227,6 +234,8 @@ export class AuthManager {
     await writeFile(tmpFile, JSON.stringify(data, null, 2), { mode: 0o600 });
     await rename(tmpFile, this.keysFile);
     await secureFilePermissions(this.keysFile);
+    // #3367: Track mtime after save
+    try { this.lastKeysMtime = statSync(this.keysFile).mtimeMs; } catch { /* ignore */ }
   }
 
   /** Create a new API key. Returns the plaintext key (only shown once). */
@@ -535,6 +544,11 @@ export class AuthManager {
           return { valid: true, keyId: rotatedKey.id, rateLimited: false, tenantId: rotatedKey.tenantId };
         }
       }
+      // #3367: Trigger async reload on invalid — recovers from state dir wipe
+      setImmediate(async () => {
+        const reloaded = await this.reload();
+        if (reloaded) console.warn('[AuthManager] Keys reloaded after failed validation');
+      });
       return { valid: false, keyId: null, rateLimited: false, reason: 'invalid' };
     }
 
@@ -613,6 +627,47 @@ export class AuthManager {
     return key?.tenantId;
   }
 
+  /** #3367: Reload keys from disk if the file has changed since last load/save. */
+  async reload(): Promise<boolean> {
+    if (this.reloading) return false;
+    this.reloading = true;
+    try {
+      if (!existsSync(this.keysFile)) {
+        console.warn('[AuthManager] keys.json disappeared — keeping in-memory keys');
+        this.lastKeysMtime = null;
+        return false;
+      }
+      if (!this._keysFileChanged()) return false;
+      console.warn('[AuthManager] keys.json changed on disk — reloading');
+      await this.load();
+      return true;
+    } finally {
+      this.reloading = false;
+    }
+  }
+
+  /** #3367: Check if keys.json mtime has changed. */
+  private _keysFileChanged(): boolean {
+    try {
+      const currentMtime = statSync(this.keysFile).mtimeMs;
+      if (this.lastKeysMtime === null || currentMtime > this.lastKeysMtime) {
+        this.lastKeysMtime = currentMtime;
+        return true;
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  }
+
+  /** #3367: Health check — false if keys.json missing but in-memory keys exist. */
+  isHealthy(): boolean {
+    if (this.store.keys.length > 0 && !existsSync(this.keysFile)) {
+      return false;
+    }
+    return true;
+  }
+
   /** Hash a key with SHA-256. */
   static hashKey(key: string): string {
     return createHash('sha256').update(key).digest('hex');