feat(server): prefix-aware inline prefix-cache eviction (#452)

The inline prefix cache evicted by pure LRU, which can drop a short,
broadly-shared ancestor prefix (e.g. system prompt plus first turn, reused by
many later branches) while keeping a long, conversation-specific leaf snapshot
that nothing else reuses. The next branch that needs the shared ancestor then
re-prefills it.

Make eviction prefix-aware: prefer evicting the oldest leaf (an entry whose
tokens are not a strict prefix of any other live entry) so shared ancestors stay
resident. Falls back to plain LRU when no ancestor structure exists.

This is not strictly better than LRU. It keeps the frozen shallow ancestors plus
the current deepest entry, while LRU keeps the N most-recent entries. It wins
when later branches reuse an early shared root (the agentic system-prompt
pattern) and can lose when a branch reuses a recent but non-current prefix that
LRU would still hold. Linear conversations are unaffected: both keep the deepest
entry.

The policy is a pure free function select_inline_evict_victim over the cached
prefixes in LRU order; inline entries now carry their prefix tokens so the leaf
test can run. Contained to prefix_cache; no backend or request-protocol change.
Adds model-free unit tests for the policy.

Author: cheese-cakee

server/src/server/prefix_cache.cpp

@@ -140,6 +140,38 @@ PrefixHash hash_prefix(const int32_t * ids, int count) {
     return h;
 }
 
+// ─── Prefix-aware eviction ──────────────────────────────────────────────
+
+static bool is_strict_prefix(const std::vector<int32_t> & a,
+                             const std::vector<int32_t> & b) {
+    // True iff `a` is a strict (shorter) prefix of `b`.
+    if (a.size() >= b.size()) return false;
+    return std::equal(a.begin(), a.end(), b.begin());
+}
+
+int select_inline_evict_victim(const std::vector<const std::vector<int32_t> *> & ids_lru) {
+    const int n = (int)ids_lru.size();
+    if (n <= 0) return 0;
+    // Oldest-first scan: evict the first entry that is not a strict prefix of any
+    // other entry (a leaf). Shared ancestor prefixes are thereby kept resident.
+    for (int i = 0; i < n; i++) {
+        bool is_ancestor = false;
+        for (int j = 0; j < n; j++) {
+            if (j == i) continue;
+            if (is_strict_prefix(*ids_lru[i], *ids_lru[j])) { is_ancestor = true; break; }
+        }
+        if (!is_ancestor) return i;  // oldest leaf
+    }
+    return 0;  // unreachable (the longest entry is always a leaf); pure-LRU fallback
+}
+
+int select_inline_evict_victim(const std::vector<std::vector<int32_t>> & ids_lru) {
+    std::vector<const std::vector<int32_t> *> ptrs;
+    ptrs.reserve(ids_lru.size());
+    for (const auto & v : ids_lru) ptrs.push_back(&v);
+    return select_inline_evict_victim(ptrs);
+}
+
 // ─── PrefixCache ────────────────────────────────────────────────────────
 
 PrefixCache::PrefixCache(int cap, const Tokenizer & tokenizer)
@@ -171,9 +203,9 @@ int PrefixCache::find_entry(const PrefixHash & h) const {
 
 void PrefixCache::move_to_end(int idx) {
     if (idx < 0 || idx >= (int)entries_.size()) return;
-    auto e = entries_[idx];
+    auto e = std::move(entries_[idx]);
     entries_.erase(entries_.begin() + idx);
-    entries_.push_back(e);
+    entries_.push_back(std::move(e));
 }
 
 int PrefixCache::find_full_entry(const PrefixHash & h) const {
@@ -235,10 +267,22 @@ std::pair<int, int> PrefixCache::prepare_inline_snap(
 
     int slot;
     if ((int)entries_.size() >= cap_) {
-        // At capacity — reserve the LRU slot without evicting yet.
-        pending_evict_key_ = entries_.front().hash;
+        // At capacity — reserve a slot without evicting yet. Prefix-aware: prefer
+        // the oldest leaf so shared ancestor prefixes (reused by later branches)
+        // stay resident. entries_ is already in LRU order (front = oldest).
+        std::vector<const std::vector<int32_t> *> ids_lru;
+        ids_lru.reserve(entries_.size());
+        for (const auto & e : entries_) ids_lru.push_back(&e.ids);
+        int victim = select_inline_evict_victim(ids_lru);
+        pending_evict_key_ = entries_[victim].hash;
         has_pending_evict_ = true;
-        slot = entries_.front().slot;
+        slot = entries_[victim].slot;
+        if (victim != 0) {
+            std::fprintf(stderr,
+                "[pc] prefix-aware evict: victim idx=%d (len=%zu) kept oldest "
+                "ancestor (len=%zu)\n",
+                victim, entries_[victim].ids.size(), entries_.front().ids.size());
+        }
     } else {
         slot = next_slot_;
         next_slot_ = (next_slot_ + 1) % cap_;
@@ -278,7 +322,8 @@ void PrefixCache::confirm_inline_snap(int slot, int target_cut,
     }
 
     auto key = hash_prefix(prompt_ids.data(), target_cut);
-    entries_.push_back({key, slot});
+    std::vector<int32_t> ids(prompt_ids.begin(), prompt_ids.begin() + target_cut);
+    entries_.push_back({key, slot, std::move(ids)});
     entries_size_count_.fetch_add(1, std::memory_order_relaxed);
     std::fprintf(stderr, "[pc] inline-snap committed slot=%d prefix_len=%d\n",
                  slot, target_cut);

server/src/server/prefix_cache.h

@@ -45,6 +45,18 @@ std::vector<int> find_all_boundaries(const std::vector<int32_t> & ids,
 using PrefixHash = std::array<uint8_t, 16>;
 PrefixHash hash_prefix(const int32_t * ids, int count);
 
+// Prefix-aware inline eviction policy. Given the cached prefixes in LRU order
+// (index 0 = oldest), return the index of the eviction victim: the oldest entry
+// whose ids are NOT a strict prefix of any other entry's ids (a "leaf"). Keeping
+// shared ancestor prefixes resident avoids re-prefilling them for later branches.
+// Returns 0 (pure-LRU fallback) when ids_lru is empty or, impossibly, no leaf
+// is found. Pure and model-free so it can be unit-tested without a PrefixCache.
+// The pointer overload is the core (the caller passes pointers into its own
+// entries so no token vectors are copied); the value overload is a convenience
+// wrapper for tests.
+int select_inline_evict_victim(const std::vector<const std::vector<int32_t> *> & ids_lru);
+int select_inline_evict_victim(const std::vector<std::vector<int32_t>> & ids_lru);
+
 // ─── Prefix cache entry ─────────────────────────────────────────────────
 
 struct FullCacheEntry {
@@ -139,8 +151,9 @@ class PrefixCache {
     // LRU for inline prefix cache: ordered map of hash → slot.
     // We use a vector to maintain insertion order (front = oldest).
     struct LruEntry {
-        PrefixHash hash;
-        int        slot;
+        PrefixHash           hash;
+        int                  slot;
+        std::vector<int32_t> ids;  // prefix tokens [0, target_cut) for prefix-aware eviction
     };
     std::vector<LruEntry> entries_;
     int next_slot_ = 0;

server/test/test_server_unit.cpp

@@ -1242,6 +1242,40 @@ static void test_find_boundaries_empty() {
     TEST_ASSERT(bounds.empty());
 }
 
+// ── Prefix-aware eviction policy (model-free) ───────────────────────────
+
+static void test_evict_empty_is_zero() {
+    std::vector<std::vector<int32_t>> ids;
+    TEST_ASSERT(select_inline_evict_victim(ids) == 0);
+}
+
+static void test_evict_single_is_zero() {
+    std::vector<std::vector<int32_t>> ids = {{1, 2, 3}};
+    TEST_ASSERT(select_inline_evict_victim(ids) == 0);
+}
+
+static void test_evict_chain_keeps_ancestors() {
+    // Oldest-first chain: [s] < [s,a] < [s,a,b]. Only the longest is a leaf, so
+    // the short shared ancestors are kept and the victim is the deepest entry.
+    std::vector<std::vector<int32_t>> ids = {{9}, {9, 1}, {9, 1, 2}};
+    TEST_ASSERT(select_inline_evict_victim(ids) == 2);
+}
+
+static void test_evict_unrelated_falls_back_to_lru() {
+    // No prefix relation: all are leaves, so evict the oldest (index 0).
+    std::vector<std::vector<int32_t>> ids = {{1, 1}, {2, 2}, {3, 3}};
+    TEST_ASSERT(select_inline_evict_victim(ids) == 0);
+}
+
+static void test_evict_branch_spares_shared_root() {
+    // [s] is an ancestor of both branches, so it is never the victim; the oldest
+    // leaf ([s,a] at index 1) is evicted instead.
+    std::vector<std::vector<int32_t>> ids = {{9}, {9, 1}, {9, 2}};
+    int v = select_inline_evict_victim(ids);
+    TEST_ASSERT(v == 1);
+    TEST_ASSERT(v != 0);  // the shared root must be spared
+}
+
 // ═══════════════════════════════════════════════════════════════════════
 // PFlash config tests (model-free)
 // ═══════════════════════════════════════════════════════════════════════
@@ -4054,6 +4088,11 @@ int main() {
     RUN_TEST(test_hash_prefix_different_lengths);
     RUN_TEST(test_hash_prefix_empty);
     RUN_TEST(test_find_boundaries_empty);
+    RUN_TEST(test_evict_empty_is_zero);
+    RUN_TEST(test_evict_single_is_zero);
+    RUN_TEST(test_evict_chain_keeps_ancestors);
+    RUN_TEST(test_evict_unrelated_falls_back_to_lru);
+    RUN_TEST(test_evict_branch_spares_shared_root);
 
     std::fprintf(stderr, "\n── PFlash config ──\n");
     RUN_TEST(test_pflash_config_defaults);