fix(router): scope clientside-credential deployments per-key + prune stale ones

A request carrying a clientside api_key gets a synthetic per-key deployment
upserted into the shared Router model_list (under the public model group name,
team_id=None). Two defects followed (upstream PR BerriAI#8966; upstream issue BerriAI#17115,
auto-closed stale; reproduced as JuliaHub BerriAI#22864):

1. Cross-user key leak: a later request with NO key could be load-balanced
   (simple-shuffle) onto that synthetic deployment and reuse the first caller's
   key upstream.
2. Unbounded growth: each distinct key appended a permanent entry, never removed.

Fix:
- Tag synthetic deployments (model_info["clientside_credential"]) and add
  is_clientside_credential_deployment().
- In _common_checks_available_deployment, scope selection per-key: a request
  with no clientside credential sees only config deployments; a request with one
  sees config deployments plus only its own matching synthetic.
- Opportunistically prune (throttled, synchronous, race-free on the event loop)
  synthetic deployments idle past an env-configurable TTL, with an LRU max-count
  backstop, on each keyed request via _handle_clientside_credential.

Adds deterministic regression tests for both the leak and the prune.

Author: tanmaykm

litellm/router.py

@@ -14,6 +14,7 @@
 import inspect
 import json
 import logging
+import os
 import re
 import threading
 import time
@@ -105,8 +106,10 @@
 )
 from litellm.router_utils.client_initalization_utils import InitalizeCachedClient
 from litellm.router_utils.clientside_credential_handler import (
+    clientside_credential_keys,
     get_dynamic_litellm_params,
     is_clientside_credential,
+    is_clientside_credential_deployment,
 )
 from litellm.router_utils.common_utils import (
     filter_team_based_models,
@@ -519,6 +522,19 @@ def __init__(
             self.allowed_fails = litellm.allowed_fails
         self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
         self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
+        # JH: bound growth of synthetic per-key clientside-credential deployments
+        # (created by _handle_clientside_credential and never otherwise removed).
+        self.clientside_credential_ttl = int(
+            os.getenv("LITELLM_CLIENTSIDE_CREDENTIAL_TTL", 3600)
+        )  # evict a synthetic deployment idle (unused) for this many seconds
+        self.clientside_credential_max_count = int(
+            os.getenv("LITELLM_CLIENTSIDE_CREDENTIAL_MAX_COUNT", 1000)
+        )  # hard LRU cap as a memory backstop
+        self._clientside_prune_interval = int(
+            os.getenv("LITELLM_CLIENTSIDE_PRUNE_INTERVAL", 60)
+        )  # min seconds between sweeps (throttle)
+        self.clientside_credential_last_used: Dict[str, float] = {}
+        self._last_clientside_prune: float = 0.0
         self.disable_cooldowns = disable_cooldowns
         self.enable_health_check_routing = enable_health_check_routing
         self.enable_weighted_failover = enable_weighted_failover
@@ -2908,12 +2924,17 @@ def _handle_clientside_credential(
         original_model_id = model_info.get("id")
         model_info["id"] = _model_id
         model_info["original_model_id"] = original_model_id
+        model_info["clientside_credential"] = True  # mark synthetic per-key deployment (JH security fix)
         deployment_pydantic_obj = Deployment(
             model_name=model_group,
             litellm_params=LiteLLM_Params(**dynamic_litellm_params),
             model_info=model_info,
         )
         self.upsert_deployment(deployment=deployment_pydantic_obj)  # add new deployment to router
+        # JH: track last-use and opportunistically prune stale synthetic
+        # deployments so the shared model_list does not grow unbounded.
+        self.clientside_credential_last_used[_model_id] = time.time()
+        self._maybe_prune_clientside_deployments()
         return deployment_pydantic_obj
 
     @staticmethod
@@ -8257,6 +8278,45 @@ def delete_deployment(self, id: str) -> Optional[Deployment]:
         except Exception:
             return None
 
+    def _maybe_prune_clientside_deployments(self) -> None:
+        """
+        Bound the growth of synthetic per-key clientside-credential deployments
+        created by _handle_clientside_credential. Throttled, synchronous sweep:
+          - evict any synthetic unused for > clientside_credential_ttl seconds, then
+          - if still above clientside_credential_max_count, LRU-evict the oldest.
+
+        Synchronous and iterates a model_list snapshot, so it cannot interleave
+        with synchronous deployment selection on the same event loop.
+        """
+        now = time.time()
+        if now - self._last_clientside_prune < self._clientside_prune_interval:
+            return
+        self._last_clientside_prune = now
+
+        survivors: List[Tuple[str, float]] = []  # (model_id, last_used) for kept synthetics
+        for d in [m for m in self.model_list if is_clientside_credential_deployment(m)]:
+            mid = (d.get("model_info") or {}).get("id")
+            if mid is None:
+                continue
+            last_used = self.clientside_credential_last_used.get(mid)
+            if last_used is None:
+                # First time we see it here (e.g. created before tracking) —
+                # lazily stamp and treat as fresh rather than evicting blindly.
+                self.clientside_credential_last_used[mid] = now
+                survivors.append((mid, now))
+            elif now - last_used > self.clientside_credential_ttl:
+                self.delete_deployment(mid)
+                self.clientside_credential_last_used.pop(mid, None)
+            else:
+                survivors.append((mid, last_used))
+
+        if len(survivors) > self.clientside_credential_max_count:
+            survivors.sort(key=lambda t: t[1])  # oldest first
+            n_evict = len(survivors) - self.clientside_credential_max_count
+            for mid, _ in survivors[:n_evict]:
+                self.delete_deployment(mid)
+                self.clientside_credential_last_used.pop(mid, None)
+
     def _get_router_deployment_budget_limiter(
         self,
     ) -> Optional[RouterBudgetLimiting]:
@@ -10112,6 +10172,29 @@ def _common_checks_available_deployment(
         ## get healthy deployments
         ### get all deployments
         healthy_deployments = self._get_all_deployments(model_name=model, team_id=request_team_id)
+
+        # SECURITY (JH fix; root cause upstream PR #8966, upstream issue #17115):
+        # Synthetic per-key clientside-credential deployments are upserted into
+        # the shared model_list and would otherwise be candidates for EVERY
+        # request to this model group. Drop them unless THIS request carries a
+        # matching clientside credential: a no-key request sees none (else it
+        # could be routed onto and reuse another caller's baked key); a keyed
+        # request sees only its OWN synthetic deployment.
+        _req_kwargs = request_kwargs or {}
+        _req_is_clientside = is_clientside_credential(_req_kwargs)
+        healthy_deployments = [
+            d
+            for d in healthy_deployments
+            if not is_clientside_credential_deployment(d)
+            or (
+                _req_is_clientside
+                and all(
+                    (d.get("litellm_params", {}) or {}).get(k) == _req_kwargs.get(k)
+                    for k in clientside_credential_keys
+                )
+            )
+        ]
+
         _pre_model_access_group_filter_len = len(healthy_deployments)
         healthy_deployments = self._filter_deployments_by_model_access_groups(
             model=model,

litellm/router_utils/clientside_credential_handler.py

@@ -73,6 +73,21 @@ def is_clientside_credential(request_kwargs: dict) -> bool:
     return any(key in request_kwargs for key in clientside_credential_keys)
 
 
+def is_clientside_credential_deployment(model: dict) -> bool:
+    """
+    True if this deployment is a synthetic per-request clientside-credential
+    deployment created by Router._handle_clientside_credential. Such deployments
+    bake a single caller's api_key/api_base into litellm_params and must not be
+    selected for requests that did not supply that same clientside credential
+    (otherwise another caller's key would be reused).
+    """
+    model_info = model.get("model_info") or {}
+    return bool(
+        model_info.get("clientside_credential")
+        or model_info.get("original_model_id")  # redundant fallback marker
+    )
+
+
 def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> dict:
     """
     Generate a unique model_id for the deployment.

tests/local_testing/test_router_utils.py

@@ -427,6 +427,97 @@ def test_router_handle_clientside_credential():
     assert len(router.get_model_list()) == 2
 
 
+def test_router_clientside_credential_per_key_scoped_selection():
+    """
+    Regression for the clientside-credential synthetic-deployment leak
+    (root cause upstream PR #8966; upstream issue #17115; JuliaHub #22864).
+
+    When a request supplies a clientside api_key, the router upserts a synthetic
+    per-key deployment into the shared model_list under the public model group
+    name. Without scoping, a later request could be load-balanced onto another
+    caller's synthetic deployment and reuse their key. The fix scopes selection:
+    a no-key request sees only config deployments; a keyed request sees config
+    deployments plus only its own matching synthetic.
+
+    Deterministic: asserts on the candidate set, independent of random.choice.
+    """
+    base = {
+        "model_name": "claude-opus-4-8",
+        "litellm_params": {
+            "model": "anthropic/claude-opus-4-8",
+            "api_key": "CONFIGKEY",
+        },
+        "model_info": {"id": "base-1"},
+    }
+    router = Router(model_list=[base])
+
+    dep_a = router._handle_clientside_credential(
+        deployment=base,
+        kwargs={"api_key": "USERKEY_A", "metadata": {"model_group": "claude-opus-4-8"}},
+        function_name="acompletion",
+    )
+    dep_b = router._handle_clientside_credential(
+        deployment=base,
+        kwargs={"api_key": "USERKEY_B", "metadata": {"model_group": "claude-opus-4-8"}},
+        function_name="acompletion",
+    )
+    assert dep_a.litellm_params.api_key == "USERKEY_A"
+    assert dep_b.litellm_params.api_key == "USERKEY_B"
+    assert len(router.get_model_list()) == 3  # base + two synthetics persisted
+
+    def candidate_keys(request_kwargs):
+        _, healthy = router._common_checks_available_deployment(
+            model="claude-opus-4-8", request_kwargs=request_kwargs
+        )
+        return sorted(d["litellm_params"].get("api_key") for d in healthy)
+
+    # (1) NO-key request: ONLY the config deployment.
+    assert candidate_keys({}) == ["CONFIGKEY"]
+    # (2) Keyed (USERKEY_A): config + its OWN synthetic, NOT USERKEY_B's.
+    assert candidate_keys({"api_key": "USERKEY_A"}) == ["CONFIGKEY", "USERKEY_A"]
+    # (3) Symmetric for USERKEY_B.
+    assert candidate_keys({"api_key": "USERKEY_B"}) == ["CONFIGKEY", "USERKEY_B"]
+
+
+def test_router_prunes_stale_clientside_credential_deployments():
+    """
+    Synthetic per-key clientside-credential deployments unused beyond a TTL are
+    evicted, bounding memory growth (JuliaHub #22864; upstream PR #8966 / #17115).
+    Deterministic: backdates last-used timestamps rather than sleeping.
+    """
+    base = {
+        "model_name": "claude-opus-4-8",
+        "litellm_params": {
+            "model": "anthropic/claude-opus-4-8",
+            "api_key": "CONFIGKEY",
+        },
+        "model_info": {"id": "base-1"},
+    }
+    router = Router(model_list=[base])
+    router.clientside_credential_ttl = 3600
+    router._clientside_prune_interval = 0  # disable throttle for the test
+
+    def mk(key):
+        dep = router._handle_clientside_credential(
+            deployment=base,
+            kwargs={"api_key": key, "metadata": {"model_group": "claude-opus-4-8"}},
+            function_name="acompletion",
+        )
+        return dep.model_info.id
+
+    id_a, id_b = mk("USERKEY_A"), mk("USERKEY_B")
+    assert len(router.get_model_list()) == 3
+
+    router.clientside_credential_last_used[id_a] = 0.0  # epoch -> stale
+    router.clientside_credential_last_used[id_b] = time.time()  # fresh
+    router._maybe_prune_clientside_deployments()
+
+    ids = {d["model_info"]["id"] for d in router.get_model_list()}
+    assert id_a not in ids  # stale synthetic evicted
+    assert id_b in ids and "base-1" in ids  # fresh synthetic + config kept
+    assert id_a not in router.clientside_credential_last_used  # side dict cleaned
+
+
 def test_router_get_async_openai_model_client():
     router = Router(
         model_list=[