feat: Add cache persistence and fix dashboard config

Cache Persistence:
- Integrate CacheStorage into HybridCache for persistent disk storage
- Load existing entries from ~/.empathy/cache/responses.json on startup
- Persist new entries to disk after each put operation
- All 56 cache unit tests pass

Dashboard Config Fix:
- Add extra="ignore" to Pydantic Settings to allow extra env vars
- Fixes ValidationError preventing test collection

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: silversurfer562

dashboard/backend/config.py

@@ -94,6 +94,7 @@ class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"
         case_sensitive = False
+        extra = "ignore"  # Ignore extra environment variables (like API keys)
 
     def model_post_init(self, __context) -> None:
         """Post-initialization validation and setup."""

src/empathy_os/cache/hybrid.py

@@ -15,11 +15,13 @@
 import hashlib
 import logging
 import time
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
 from .base import BaseCache, CacheEntry, CacheStats
+from .storage import CacheStorage
 
 if TYPE_CHECKING:
     from sentence_transformers import SentenceTransformer
@@ -77,6 +79,7 @@ def __init__(
         similarity_threshold: float = 0.95,
         model_name: str = "all-MiniLM-L6-v2",
         device: str = "cpu",
+        cache_dir: Path | None = None,
     ):
         """Initialize hybrid cache.
 
@@ -87,6 +90,7 @@ def __init__(
             similarity_threshold: Semantic similarity threshold (0.0-1.0, default: 0.95).
             model_name: Sentence transformer model (default: all-MiniLM-L6-v2).
             device: Device for embeddings ("cpu" or "cuda").
+            cache_dir: Directory for persistent cache storage (default: ~/.empathy/cache/).
 
         """
         super().__init__(max_size_mb, default_ttl)
@@ -106,9 +110,16 @@ def __init__(
         self._model: SentenceTransformer | None = None
         self._load_model()
 
+        # Initialize persistent storage
+        self._storage = CacheStorage(cache_dir=cache_dir, max_disk_mb=max_size_mb)
+
+        # Load existing entries from storage into memory caches
+        self._load_from_storage()
+
         logger.info(
             f"HybridCache initialized (model: {model_name}, threshold: {similarity_threshold}, "
-            f"device: {device}, max_memory: {max_memory_mb}MB)"
+            f"device: {device}, max_memory: {max_memory_mb}MB, "
+            f"loaded: {len(self._hash_cache)} entries from disk)"
         )
 
     def _load_model(self) -> None:
@@ -131,6 +142,34 @@ def _load_model(self) -> None:
             logger.warning("Falling back to hash-only mode")
             self._model = None
 
+    def _load_from_storage(self) -> None:
+        """Load cached entries from persistent storage into memory caches."""
+        try:
+            # Get all non-expired entries from storage
+            entries = self._storage.get_all()
+
+            if not entries:
+                logger.debug("No cached entries found in storage")
+                return
+
+            # Populate hash cache
+            for entry in entries:
+                self._hash_cache[entry.key] = entry
+                self._access_times[entry.key] = entry.timestamp
+
+            logger.info(f"Loaded {len(entries)} entries from persistent storage into hash cache")
+
+            # Populate semantic cache if model available
+            if self._model is not None:
+                logger.debug("Generating embeddings for cached prompts...")
+                # Note: We don't have the original prompts, so semantic cache
+                # will be populated on-demand as cache hits occur
+                # This is acceptable since semantic matching is secondary to hash matching
+                logger.debug("Semantic cache will be populated on-demand from hash hits")
+
+        except Exception as e:
+            logger.warning(f"Failed to load cache from storage: {e}, starting with empty cache")
+
     def get(
         self,
         workflow: str,
@@ -257,7 +296,7 @@ def put(
         response: Any,
         ttl: int | None = None,
     ) -> None:
-        """Store response in both hash and semantic caches.
+        """Store response in both hash and semantic caches, and persist to disk.
 
         Args:
             workflow: Workflow name.
@@ -295,22 +334,43 @@ def put(
             prompt_embedding = self._model.encode(prompt, convert_to_numpy=True)
             self._semantic_cache.append((prompt_embedding, entry))
 
-        logger.debug(
-            f"Cache PUT (hybrid): {workflow}/{stage} "
-            f"(hash_entries: {len(self._hash_cache)}, "
-            f"semantic_entries: {len(self._semantic_cache)})"
-        )
+        # Persist to disk storage
+        try:
+            self._storage.put(entry)
+            logger.debug(
+                f"Cache PUT (hybrid): {workflow}/{stage} "
+                f"(hash_entries: {len(self._hash_cache)}, "
+                f"semantic_entries: {len(self._semantic_cache)}, "
+                f"persisted: True)"
+            )
+        except Exception as e:
+            logger.warning(f"Failed to persist cache entry to disk: {e}")
+            logger.debug(
+                f"Cache PUT (hybrid): {workflow}/{stage} "
+                f"(hash_entries: {len(self._hash_cache)}, "
+                f"semantic_entries: {len(self._semantic_cache)}, "
+                f"persisted: False)"
+            )
 
     def clear(self) -> None:
-        """Clear all cached entries."""
+        """Clear all cached entries from memory and disk."""
         hash_count = len(self._hash_cache)
         semantic_count = len(self._semantic_cache)
 
         self._hash_cache.clear()
         self._access_times.clear()
         self._semantic_cache.clear()
 
-        logger.info(f"Cache cleared (hash: {hash_count}, semantic: {semantic_count} entries)")
+        # Clear persistent storage
+        try:
+            storage_count = self._storage.clear()
+            logger.info(
+                f"Cache cleared (hash: {hash_count}, semantic: {semantic_count}, "
+                f"storage: {storage_count} entries)"
+            )
+        except Exception as e:
+            logger.warning(f"Failed to clear persistent storage: {e}")
+            logger.info(f"Cache cleared (hash: {hash_count}, semantic: {semantic_count} entries)")
 
     def get_stats(self) -> CacheStats:
         """Get cache statistics."""

tests/unit/cache/test_hybrid_cache.py

@@ -82,7 +82,7 @@ def encode_side_effect(text, **kwargs):
         assert cache.stats.hits == 1
 
     @patch("sentence_transformers.SentenceTransformer")
-    def test_semantic_cache_miss_below_threshold(self, mock_st):
+    def test_semantic_cache_miss_below_threshold(self, mock_st, tmp_path):
         """Test semantic cache miss when similarity below threshold."""
         mock_model = Mock()
         mock_st.return_value = mock_model
@@ -100,7 +100,8 @@ def encode_side_effect(text, **kwargs):
 
         mock_model.encode.side_effect = encode_side_effect
 
-        cache = HybridCache(similarity_threshold=0.95)
+        # Use isolated cache directory
+        cache = HybridCache(similarity_threshold=0.95, cache_dir=tmp_path / "cache")
 
         # Store
         cache.put("code-review", "scan", "prompt1", "sonnet", {"id": 1})
@@ -139,7 +140,7 @@ def test_semantic_only_matches_same_workflow_stage_model(self, mock_st):
         assert result is None
 
     @patch("sentence_transformers.SentenceTransformer")
-    def test_semantic_hit_adds_to_hash_cache(self, mock_st):
+    def test_semantic_hit_adds_to_hash_cache(self, mock_st, tmp_path):
         """Test that semantic hits are added to hash cache for future fast lookups."""
         mock_model = Mock()
         mock_st.return_value = mock_model
@@ -148,7 +149,8 @@ def test_semantic_hit_adds_to_hash_cache(self, mock_st):
         embedding = np.array([1.0, 0.0, 0.0])
         mock_model.encode.return_value = embedding
 
-        cache = HybridCache(similarity_threshold=0.90)
+        # Use isolated cache directory
+        cache = HybridCache(similarity_threshold=0.90, cache_dir=tmp_path / "cache")
 
         # Store original
         cache.put("code-review", "scan", "prompt1", "sonnet", {"id": 1})
@@ -221,13 +223,14 @@ def test_lru_eviction_both_caches(self, mock_st):
         assert len(cache._semantic_cache) < 200
 
     @patch("sentence_transformers.SentenceTransformer")
-    def test_size_info(self, mock_st):
+    def test_size_info(self, mock_st, tmp_path):
         """Test cache size information."""
         mock_model = Mock()
         mock_st.return_value = mock_model
         mock_model.encode.return_value = np.array([1.0, 0.0, 0.0])
 
-        cache = HybridCache(similarity_threshold=0.95)
+        # Use isolated cache directory to avoid loading existing entries
+        cache = HybridCache(similarity_threshold=0.95, cache_dir=tmp_path / "cache")
 
         # Add entries
         for i in range(10):