perf(pdf-server): cap total cache bytes with LRU eviction

The module-level sharedPdfCache could grow unbounded within the 60s
lifetime window under a burst of distinct URLs. Track running total
bytes and evict least-recently-used entries on insert when it would
exceed CACHE_MAX_TOTAL_BYTES (256MB). getCacheEntry now bumps the
accessed entry to the end of insertion order so eviction targets the
LRU entry rather than the oldest insert.

createPdfCache takes an optional maxTotalBytes for testability.

Author: ochafik

examples/pdf-server/server.test.ts

@@ -71,6 +71,72 @@ describe("PDF Cache with Timeouts", () => {
     });
   });
 
+  describe("byte-cap LRU eviction", () => {
+    const tenBytes = new Uint8Array(10);
+
+    async function fill(cache: PdfCache, url: string): Promise<void> {
+      const m = spyOn(globalThis, "fetch").mockResolvedValueOnce(
+        new Response(tenBytes, { status: 200 }),
+      );
+      try {
+        await cache.readPdfRange(url, 0, 1024);
+      } finally {
+        m.mockRestore();
+      }
+    }
+
+    it("evicts least-recently-used entry when total exceeds cap", async () => {
+      const cache = createPdfCache(25);
+      try {
+        await fill(cache, "https://arxiv.org/pdf/a");
+        await fill(cache, "https://arxiv.org/pdf/b");
+        expect(cache.getCacheSize()).toBe(2);
+
+        // Touch A so B becomes least-recently-used
+        await cache.readPdfRange("https://arxiv.org/pdf/a", 0, 1);
+
+        // Inserting C (10B) pushes total to 30 > 25 → evict LRU (B)
+        await fill(cache, "https://arxiv.org/pdf/c");
+        expect(cache.getCacheSize()).toBe(2);
+
+        // A and C still served from cache; B re-fetches
+        const m = spyOn(globalThis, "fetch").mockResolvedValue(
+          new Response(tenBytes, { status: 200 }),
+        );
+        try {
+          await cache.readPdfRange("https://arxiv.org/pdf/a", 0, 1);
+          await cache.readPdfRange("https://arxiv.org/pdf/c", 0, 1);
+          expect(m).toHaveBeenCalledTimes(0);
+          await cache.readPdfRange("https://arxiv.org/pdf/b", 0, 1);
+          expect(m).toHaveBeenCalledTimes(1);
+        } finally {
+          m.mockRestore();
+        }
+      } finally {
+        cache.clearCache();
+      }
+    });
+
+    it("evicts multiple entries if a single insert exceeds the cap", async () => {
+      const cache = createPdfCache(25);
+      try {
+        await fill(cache, "https://arxiv.org/pdf/a");
+        await fill(cache, "https://arxiv.org/pdf/b");
+        const big = spyOn(globalThis, "fetch").mockResolvedValueOnce(
+          new Response(new Uint8Array(20), { status: 200 }),
+        );
+        try {
+          await cache.readPdfRange("https://arxiv.org/pdf/big", 0, 1024);
+        } finally {
+          big.mockRestore();
+        }
+        expect(cache.getCacheSize()).toBe(1);
+      } finally {
+        cache.clearCache();
+      }
+    });
+  });
+
   describe("readPdfRange caching behavior", () => {
     const testUrl = "https://arxiv.org/pdf/test-pdf";
     const testData = new Uint8Array([0x25, 0x50, 0x44, 0x46]); // %PDF header

examples/pdf-server/server.ts

@@ -84,6 +84,9 @@ export const CACHE_MAX_LIFETIME_MS = 60_000; // 60 seconds
 /** Max size for cached PDFs (defensive limit to prevent memory exhaustion) */
 export const CACHE_MAX_PDF_SIZE_BYTES = 50 * 1024 * 1024; // 50MB
 
+/** Max total bytes across all cache entries; oldest evicted first when exceeded. */
+export const CACHE_MAX_TOTAL_BYTES = 256 * 1024 * 1024; // 256MB
+
 /** Allowed local file paths (CLI args + file roots — read access). */
 export const allowedLocalFiles = new Set<string>();
 
@@ -696,15 +699,19 @@ export interface PdfCache {
  * - CACHE_INACTIVITY_TIMEOUT_MS of no access (resets on each access)
  * - CACHE_MAX_LIFETIME_MS from creation (absolute timeout)
  */
-export function createPdfCache(): PdfCache {
+export function createPdfCache(
+  maxTotalBytes: number = CACHE_MAX_TOTAL_BYTES,
+): PdfCache {
   const cache = new Map<string, CacheEntry>();
+  let totalBytes = 0;
 
   /** Delete a cache entry and clear its timers */
   function deleteCacheEntry(url: string): void {
     const entry = cache.get(url);
     if (entry) {
       clearTimeout(entry.inactivityTimer);
       clearTimeout(entry.maxLifetimeTimer);
+      totalBytes -= entry.data.length;
       cache.delete(url);
     }
   }
@@ -720,6 +727,10 @@ export function createPdfCache(): PdfCache {
       deleteCacheEntry(url);
     }, CACHE_INACTIVITY_TIMEOUT_MS);
 
+    // Move to end of insertion order so size-cap eviction is LRU.
+    cache.delete(url);
+    cache.set(url, entry);
+
     return entry.data;
   }
 
@@ -728,6 +739,12 @@ export function createPdfCache(): PdfCache {
     // Clear any existing entry first
     deleteCacheEntry(url);
 
+    // Evict least-recently-used entries until under the byte cap.
+    for (const oldest of cache.keys()) {
+      if (totalBytes + data.length <= maxTotalBytes) break;
+      deleteCacheEntry(oldest);
+    }
+
     const entry: CacheEntry = {
       data,
       createdAt: Date.now(),
@@ -740,6 +757,7 @@ export function createPdfCache(): PdfCache {
     };
 
     cache.set(url, entry);
+    totalBytes += data.length;
   }
 
   /** Slice a cached or freshly-fetched full body to the requested range. */