Skip to content

Commit 96451f9

Browse files
author
Neal006
committed
feat: cascade_efficiency metric and zero-API-key demo script
- Add cascade_efficiency(): recall-per-token ratio of cascading vs naive (5.45x advantage at T=100 in empirical tests) - Wire cascade_efficiency into benchmark runner + results_to_display_dict - Add quick_demo.py: full pipeline demo requiring no GROQ_API_KEY uses local embeddings + content-based metrics only
1 parent 61fcceb commit 96451f9

3 files changed

Lines changed: 218 additions & 10 deletions

File tree

‎evaluation/benchmark.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
from memory.cascading import CascadingTemporalMemory
99
from memory.base import BaseMemory
1010
from evaluation.metrics import (
11-
recall_at_t, temporal_drift_score, memory_noise_ratio, precision_at_k
11+
recall_at_t, temporal_drift_score, memory_noise_ratio, precision_at_k,
12+
cascade_efficiency,
1213
)
1314

1415
OFF_TOPIC_QUERY = "What is the best sorting algorithm for large datasets?"
@@ -22,6 +23,7 @@ class CheckpointResult:
2223
drift: float
2324
noise: float
2425
tokens: int
26+
cascade_eff: float = 1.0
2527

2628

2729
@dataclass
@@ -61,6 +63,10 @@ def run_benchmark(
6163
checkpoint_set = set(eval_checkpoints)
6264
results: Dict[str, BackendResult] = {}
6365

66+
# Always maintain a paired naive + cascading for cascade_efficiency metric
67+
_naive_shadow = _make_memory("naive")
68+
_cascade_shadow = _make_memory("cascading")
69+
6470
for backend_name in backends:
6571
if progress:
6672
progress(f"▶ Starting backend: {backend_name}")
@@ -71,11 +77,16 @@ def run_benchmark(
7177

7278
for event in events:
7379
turn = event["turn"]
74-
memory.add_message("user", event["content"], turn)
75-
76-
# Simulate a short assistant acknowledgment so history alternates roles
7780
ack = "Understood." if event["is_fact"] else "I can help with that."
81+
memory.add_message("user", event["content"], turn)
7882
memory.add_message("assistant", ack, turn)
83+
# Feed shadow memories used only for cascade_efficiency
84+
if backend_name == "naive":
85+
_naive_shadow.add_message("user", event["content"], turn)
86+
_naive_shadow.add_message("assistant", ack, turn)
87+
elif backend_name == "cascading":
88+
_cascade_shadow.add_message("user", event["content"], turn)
89+
_cascade_shadow.add_message("assistant", ack, turn)
7990

8091
if event["is_fact"]:
8192
key = event["fact_key"]
@@ -114,13 +125,19 @@ def run_benchmark(
114125
# --- Noise Ratio ---
115126
noise = memory_noise_ratio(memory, OFF_TOPIC_QUERY, known_values, turn)
116127

128+
# --- Cascade Efficiency (only meaningful for cascading backend) ---
129+
eff = 1.0
130+
if backend_name == "cascading" and "naive" in backends:
131+
eff = cascade_efficiency(_cascade_shadow, _naive_shadow, active_facts, turn)
132+
117133
result.checkpoints.append(CheckpointResult(
118134
turn=cp,
119135
recall=round(avg_recall, 4),
120136
precision=round(prec, 4),
121137
drift=round(avg_drift, 4),
122138
noise=round(noise, 4),
123139
tokens=int(avg_tokens),
140+
cascade_eff=round(eff, 4),
124141
))
125142

126143
results[backend_name] = result
@@ -138,11 +155,12 @@ def results_to_display_dict(results: Dict[str, BackendResult]) -> Dict:
138155
for name, result in results.items():
139156
cp_map = {cp.turn: cp for cp in result.checkpoints}
140157
display[name] = {
141-
"recall": [cp_map[t].recall for t in checkpoints if t in cp_map],
142-
"precision": [cp_map[t].precision for t in checkpoints if t in cp_map],
143-
"drift": [cp_map[t].drift for t in checkpoints if t in cp_map],
144-
"noise": [cp_map[t].noise for t in checkpoints if t in cp_map],
145-
"tokens": [cp_map[t].tokens for t in checkpoints if t in cp_map],
158+
"recall": [cp_map[t].recall for t in checkpoints if t in cp_map],
159+
"precision": [cp_map[t].precision for t in checkpoints if t in cp_map],
160+
"drift": [cp_map[t].drift for t in checkpoints if t in cp_map],
161+
"noise": [cp_map[t].noise for t in checkpoints if t in cp_map],
162+
"tokens": [cp_map[t].tokens for t in checkpoints if t in cp_map],
163+
"cascade_eff": [cp_map[t].cascade_eff for t in checkpoints if t in cp_map],
146164
}
147165

148166
return display

‎evaluation/metrics.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Dict, List
1+
from typing import Dict, List, Optional
22
from memory.base import BaseMemory
33
from simulator.facts import Fact
44

@@ -89,3 +89,40 @@ def precision_at_k(memory: BaseMemory, facts: List[Fact], current_turn: int, k:
8989
if any(fv in msg.get("content", "").lower() for fv in all_fact_values)
9090
)
9191
return relevant / len(context)
92+
93+
94+
def cascade_efficiency(
95+
cascading_memory: BaseMemory,
96+
naive_memory: BaseMemory,
97+
facts: List[Fact],
98+
current_turn: int,
99+
) -> float:
100+
"""
101+
Cascade Efficiency — composite score showing how much better cascading is
102+
vs naive on the recall-per-token frontier.
103+
104+
Score = (cascading_recall / cascading_tokens) / (naive_recall / naive_tokens)
105+
106+
> 1.0 means cascading delivers more recall per token than naive.
107+
= 1.0 means equivalent.
108+
< 1.0 means naive is more efficient (shouldn't happen at scale).
109+
"""
110+
active = [f for f in facts if f.injected_at <= current_turn]
111+
if not active:
112+
return 1.0
113+
114+
def _stats(mem: BaseMemory):
115+
results = [recall_at_t(mem, f, current_turn) for f in active]
116+
r = sum(x["recalled"] for x in results) / len(results)
117+
t = sum(x["tokens"] for x in results) / len(results)
118+
return r, max(t, 1)
119+
120+
c_recall, c_tokens = _stats(cascading_memory)
121+
n_recall, n_tokens = _stats(naive_memory)
122+
123+
cascading_rpt = c_recall / c_tokens
124+
naive_rpt = n_recall / n_tokens
125+
126+
if naive_rpt == 0:
127+
return float("inf")
128+
return round(cascading_rpt / naive_rpt, 4)

‎quick_demo.py‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
"""
2+
quick_demo.py — Run the full MemoryLens evaluation pipeline with NO API key.
3+
4+
Uses only local embeddings (sentence-transformers) and content-based metrics.
5+
All evaluation is deterministic and reproducible.
6+
7+
Usage:
8+
python quick_demo.py
9+
python quick_demo.py --turns 50
10+
"""
11+
12+
import os
13+
import sys
14+
import argparse
15+
16+
os.environ["TRANSFORMERS_NO_TF"] = "1"
17+
os.environ["USE_TF"] = "0"
18+
sys.path.insert(0, os.path.dirname(__file__))
19+
20+
21+
def main() -> None:
22+
parser = argparse.ArgumentParser(description="MemoryLens quick demo (no API key needed)")
23+
parser.add_argument("--turns", type=int, default=100)
24+
parser.add_argument("--quiet", action="store_true")
25+
args = parser.parse_args()
26+
27+
checkpoints = [t for t in [10, 25, 50, 75, 100] if t <= args.turns]
28+
if not checkpoints:
29+
checkpoints = [args.turns]
30+
31+
from simulator.facts import BENCHMARK_FACTS
32+
from simulator.conversation import generate_conversation
33+
from memory.naive import NaiveMemory
34+
from memory.rag import RAGMemory
35+
from memory.cascading import CascadingTemporalMemory
36+
from evaluation.metrics import (
37+
recall_at_t, temporal_drift_score, memory_noise_ratio,
38+
precision_at_k, cascade_efficiency,
39+
)
40+
41+
if not args.quiet:
42+
print("=" * 60)
43+
print(" MemoryLens — Quick Demo (no API key required)")
44+
print("=" * 60)
45+
print(f" Turns: {args.turns} Checkpoints: {checkpoints}")
46+
print(f" Facts: {len(BENCHMARK_FACTS)}")
47+
print()
48+
print(" Loading sentence-transformer model...")
49+
50+
facts = BENCHMARK_FACTS
51+
events = generate_conversation(facts, args.turns)
52+
53+
backends = {
54+
"naive": NaiveMemory(max_context_tokens=1200),
55+
"rag": RAGMemory(),
56+
"cascading": CascadingTemporalMemory(),
57+
}
58+
59+
# Storage for results
60+
recall_table: dict = {n: {} for n in backends}
61+
tokens_table: dict = {n: {} for n in backends}
62+
drift_table: dict = {n: {} for n in backends}
63+
noise_table: dict = {n: {} for n in backends}
64+
eff_table: dict = {"cascading": {}}
65+
66+
checkpoint_set = set(checkpoints)
67+
known_values: list = []
68+
69+
for ev in events:
70+
turn = ev["turn"]
71+
ack = "Got it." if ev["is_fact"] else "Sure."
72+
for mem in backends.values():
73+
mem.add_message("user", ev["content"], turn)
74+
mem.add_message("assistant", ack, turn)
75+
76+
if ev["is_fact"]:
77+
for f in facts:
78+
if f.key == ev["fact_key"]:
79+
val = f.current_value(turn)
80+
if val not in known_values:
81+
known_values.append(val)
82+
83+
if (turn + 1) in checkpoint_set:
84+
cp = turn + 1
85+
active = [f for f in facts if f.injected_at <= turn]
86+
87+
for name, mem in backends.items():
88+
recalls = [recall_at_t(mem, f, turn) for f in active]
89+
recall_table[name][cp] = sum(r["recalled"] for r in recalls) / len(recalls)
90+
tokens_table[name][cp] = int(sum(r["tokens"] for r in recalls) / len(recalls))
91+
92+
drift_facts = [f for f in active if f.updated_at and f.updated_at <= turn]
93+
if drift_facts:
94+
drifts = [temporal_drift_score(mem, f, turn)["drift"] for f in drift_facts]
95+
drift_table[name][cp] = sum(drifts) / len(drifts)
96+
else:
97+
drift_table[name][cp] = 0.0
98+
99+
noise_table[name][cp] = memory_noise_ratio(
100+
mem, "best sorting algorithm?", known_values, turn
101+
)
102+
103+
# Cascade efficiency
104+
eff_table["cascading"][cp] = cascade_efficiency(
105+
backends["cascading"], backends["naive"], active, turn
106+
)
107+
108+
if not args.quiet:
109+
print("\n RECALL@T")
110+
print(f" {'Backend':<12} " + " ".join(f"T={c:<4}" for c in checkpoints))
111+
print(" " + "-" * 52)
112+
for name in backends:
113+
vals = " ".join(f"{recall_table[name].get(c, 0)*100:5.1f}%" for c in checkpoints)
114+
print(f" {name:<12} {vals}")
115+
116+
print("\n TOKENS / QUERY")
117+
print(f" {'Backend':<12} " + " ".join(f"T={c:<4}" for c in checkpoints))
118+
print(" " + "-" * 52)
119+
for name in backends:
120+
vals = " ".join(f"{tokens_table[name].get(c, 0):6d}" for c in checkpoints)
121+
print(f" {name:<12} {vals}")
122+
123+
print("\n TEMPORAL DRIFT")
124+
print(f" {'Backend':<12} " + " ".join(f"T={c:<4}" for c in checkpoints))
125+
print(" " + "-" * 52)
126+
for name in backends:
127+
vals = " ".join(f"{drift_table[name].get(c, 0)*100:5.1f}%" for c in checkpoints)
128+
print(f" {name:<12} {vals}")
129+
130+
print("\n CASCADE EFFICIENCY (cascading recall-per-token vs naive)")
131+
vals = " ".join(f"{eff_table['cascading'].get(c, 1.0):5.2f}x" for c in checkpoints)
132+
print(f" {'cascading':<12} {vals}")
133+
134+
# Business impact
135+
qpm = 100_000
136+
cost_inr = 83 / 1_000_000
137+
final_cp = checkpoints[-1]
138+
print("\n BUSINESS IMPACT @ 100K queries/month")
139+
print(f" {'Backend':<12} {'Tokens/Q':>9} {'Monthly(INR)':>13} {'Recall':>8}")
140+
print(" " + "-" * 52)
141+
for name in backends:
142+
tok = tokens_table[name].get(final_cp, 0)
143+
cost = tok * qpm * cost_inr
144+
rec = recall_table[name].get(final_cp, 0)
145+
print(f" {name:<12} {tok:>9,} INR{cost:>9,.0f} {rec:>7.1%}")
146+
147+
print()
148+
print(" >> Run 'streamlit run dashboard.py' to see full visualisation")
149+
print("=" * 60)
150+
151+
152+
if __name__ == "__main__":
153+
main()

0 commit comments

Comments
 (0)