@@ -374,6 +374,62 @@ bool llama_kv_cache_tiered::evict_tokens(uint32_t n_tokens_to_evict, llama_cache
374374 return false ;
375375 }
376376
377+ // Apply semantic eviction weighting if we have a current query embedding
378+ if (!current_query_emb.empty () && !fingerprints.empty ()) {
379+ std::vector<std::pair<llama_pos, float >> candidates_with_similarity;
380+
381+ for (auto pos : candidates) {
382+ float similarity = 0 .0f ; // Default similarity if no fingerprint found
383+
384+ // Find fingerprint for this position
385+ for (const auto & fp : fingerprints) {
386+ bool found = false ;
387+ for (auto fp_pos : fp.positions ) {
388+ if (fp_pos == pos) {
389+ found = true ;
390+ break ;
391+ }
392+ }
393+ if (found) {
394+ // Compute cosine similarity (dot product since both are L2-normalized)
395+ similarity = 0 .0f ;
396+ for (size_t i = 0 ; i < current_query_emb.size () && i < fp.embedding .size (); i++) {
397+ similarity += current_query_emb[i] * fp.embedding [i];
398+ }
399+ break ;
400+ }
401+ }
402+
403+ candidates_with_similarity.emplace_back (pos, similarity);
404+ }
405+
406+ // Reorder candidates: positions with similarity < 0.3 evicted first,
407+ // positions with similarity > 0.65 moved to back of eviction queue
408+ std::sort (candidates_with_similarity.begin (), candidates_with_similarity.end (),
409+ [](const auto & a, const auto & b) {
410+ if (a.second > 0 .65f ) return false ; // Keep high similarity at back
411+ if (b.second > 0 .65f ) return true ; // Keep high similarity at back
412+ if (a.second < 0 .3f ) return true ; // Evict low similarity first
413+ if (b.second < 0 .3f ) return false ; // Evict low similarity first
414+ return a.second < b.second ; // Otherwise by similarity
415+ });
416+
417+ // Update candidates with reordered positions
418+ candidates.clear ();
419+ for (const auto & candidate : candidates_with_similarity) {
420+ candidates.push_back (candidate.first );
421+ }
422+
423+ // Count semantic eviction saves
424+ uint32_t saves = 0 ;
425+ for (const auto & candidate : candidates_with_similarity) {
426+ if (candidate.second > 0 .65f ) {
427+ saves++;
428+ }
429+ }
430+ stats.semantic_eviction_saves += saves;
431+ }
432+
377433 std::vector<llama_pos> to_warm, to_cold;
378434
379435 // Route to warm if slots available (RAM or GPU), otherwise cold
@@ -767,6 +823,145 @@ void llama_kv_cache_tiered::reset_stats() {
767823 stats.reset ();
768824}
769825
826+ void llama_kv_cache_tiered::add_fingerprint (const std::vector<llama_pos>& positions,
827+ const std::vector<float >& embedding,
828+ llama_cache_tier tier) {
829+ std::lock_guard<std::mutex> lock (mutex);
830+
831+ semantic_fingerprint fp;
832+ fp.positions = positions;
833+ fp.embedding = embedding;
834+ fp.tier = tier;
835+ fp.turn = fingerprint_turn++;
836+
837+ // Cap at 1000 entries - evict oldest when over cap
838+ if (fingerprints.size () >= 1000 ) {
839+ fingerprints.erase (fingerprints.begin ());
840+ }
841+
842+ fingerprints.push_back (fp);
843+ }
844+
845+ std::vector<llama_kv_cache_tiered::PrefetchHint> llama_kv_cache_tiered::score_fingerprints (
846+ const std::vector<float >& query_emb, int top_k, float threshold) const {
847+ std::lock_guard<std::mutex> lock (mutex);
848+
849+ std::vector<PrefetchHint> results;
850+
851+ if (fingerprints.empty () || query_emb.empty ()) {
852+ return results;
853+ }
854+
855+ // Compute cosine similarity between query_emb and each fingerprint
856+ // Since both are L2-normalized, cosine similarity = dot product
857+ for (const auto & fp : fingerprints) {
858+ // Compute dot product
859+ float dot_product = 0 .0f ;
860+ size_t min_size = std::min (query_emb.size (), fp.embedding .size ());
861+ for (size_t i = 0 ; i < min_size; ++i) {
862+ dot_product += query_emb[i] * fp.embedding [i];
863+ }
864+
865+ // Check if above threshold
866+ if (dot_product >= threshold) {
867+ PrefetchHint hint;
868+ hint.positions = fp.positions ;
869+ hint.score = dot_product;
870+ hint.current_tier = fp.tier ;
871+ results.push_back (hint);
872+ }
873+ }
874+
875+ // Sort by score descending
876+ std::sort (results.begin (), results.end (),
877+ [](const PrefetchHint& a, const PrefetchHint& b) {
878+ return a.score > b.score ;
879+ });
880+
881+ // Return top_k results
882+ if (static_cast <int >(results.size ()) > top_k) {
883+ results.resize (top_k);
884+ }
885+
886+ return results;
887+ }
888+
889+ void llama_kv_cache_tiered::set_current_query_embedding (const std::vector<float >& emb) {
890+ std::lock_guard<std::mutex> lock (mutex);
891+ current_query_emb = emb;
892+ }
893+
894+ bool llama_kv_cache_tiered::save_fingerprints_to_disk (const std::string& path) {
895+ std::lock_guard<std::mutex> lock (mutex);
896+
897+ std::ofstream file (path, std::ofstream::binary);
898+ if (!file) {
899+ return false ;
900+ }
901+
902+ // Write number of entries
903+ uint32_t n_entries = uint32_t (fingerprints.size ());
904+ file.write (reinterpret_cast <char *>(&n_entries), sizeof (uint32_t ));
905+
906+ // Write each fingerprint
907+ for (const auto & fp : fingerprints) {
908+ // Write positions count and positions
909+ uint32_t n_pos = uint32_t (fp.positions .size ());
910+ file.write (reinterpret_cast <const char *>(&n_pos), sizeof (uint32_t ));
911+ file.write (reinterpret_cast <const char *>(fp.positions .data ()), n_pos * sizeof (llama_pos));
912+
913+ // Write embedding dimension and embedding
914+ uint32_t n_embd = uint32_t (fp.embedding .size ());
915+ file.write (reinterpret_cast <const char *>(&n_embd), sizeof (uint32_t ));
916+ file.write (reinterpret_cast <const char *>(fp.embedding .data ()), n_embd * sizeof (float ));
917+
918+ // Write tier and turn
919+ file.write (reinterpret_cast <const char *>(&fp.tier ), sizeof (llama_cache_tier));
920+ file.write (reinterpret_cast <const char *>(&fp.turn ), sizeof (uint64_t ));
921+ }
922+
923+ return file.good ();
924+ }
925+
926+ bool llama_kv_cache_tiered::load_fingerprints_from_disk (const std::string& path) {
927+ std::ifstream file (path, std::ifstream::binary);
928+ if (!file) {
929+ return false ;
930+ }
931+
932+ // Read number of entries
933+ uint32_t n_entries;
934+ file.read (reinterpret_cast <char *>(&n_entries), sizeof (uint32_t ));
935+
936+ // Clear existing fingerprints
937+ fingerprints.clear ();
938+
939+ // Read each fingerprint
940+ for (uint32_t i = 0 ; i < n_entries; i++) {
941+ semantic_fingerprint fp;
942+
943+ // Read positions
944+ uint32_t n_pos;
945+ file.read (reinterpret_cast <char *>(&n_pos), sizeof (uint32_t ));
946+ fp.positions .resize (n_pos);
947+ file.read (reinterpret_cast <char *>(fp.positions .data ()), n_pos * sizeof (llama_pos));
948+
949+ // Read embedding
950+ uint32_t n_embd;
951+ file.read (reinterpret_cast <char *>(&n_embd), sizeof (uint32_t ));
952+ fp.embedding .resize (n_embd);
953+ file.read (reinterpret_cast <char *>(fp.embedding .data ()), n_embd * sizeof (float ));
954+
955+ // Read tier and turn
956+ file.read (reinterpret_cast <char *>(&fp.tier ), sizeof (llama_cache_tier));
957+ file.read (reinterpret_cast <char *>(&fp.turn ), sizeof (uint64_t ));
958+
959+ fingerprints.push_back (fp);
960+ }
961+
962+ return file.good ();
963+ }
964+
770965std::string llama_kv_cache_tiered::get_ssd_file_path (llama_pos pos) const {
771966 // Generate unique file path for token position
772967 return ssd_path + " /token_" std::to_string (pos) + " .bin"
0 commit comments