Skip to content

Commit 217fdd0

Browse files
Skiipy11claude
Skiipy11
and
claude
committed
feat: v2.5.0 — dashboard, SDK, SSE, multi-collection, retrieval fixes, docs
7 roadmap features: - Web dashboard: glassmorphism SPA at /dashboard with 7 tabs (overview, search, memories, entities, graph, clients, admin), localStorage auth - Python SDK: sync + async BrainClient with typed models, retry, httpx - SSE subscriptions: event bus + GET /subscribe endpoint for real-time streaming - Entity reclassification: heuristic auto-suggest, batch Qdrant updates, MCP tool - Automatic memory capture: relevance scorer at store time, feedback loop - Multi-collection support: parameterized qdrant.js, collection CRUD, registry - System documentation: 6 docs (architecture, operations, API ref, MCP tools, data model, configuration) 3 benchmark retrieval fixes (targeting 80%+ on LongMemEval): - Session deduplication in re-ranking (diverse session coverage) - Temporal date-range filtering with proximity boost - Query expansion with domain inference and zero-result retry Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent e433365 commit 217fdd0

30 files changed

Lines changed: 6609 additions & 35 deletions

‎api/src/index.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import { clientRouter } from './routes/client.js';
1212
import { exportRouter } from './routes/export.js';
1313
import { graphRouter } from './routes/graph.js';
1414
import { reflectRouter } from './routes/reflect.js';
15+
import { subscribeRouter } from './routes/subscribe.js';
16+
import { dashboardRouter } from './routes/dashboard.js';
17+
import { collectionsRouter } from './routes/collections.js';
1518
import { initQdrant, ensureEntityIndex } from './services/qdrant.js';
1619
import { initEmbeddings } from './services/embedders/interface.js';
1720
import { initStore, isEntityStoreAvailable, loadAllAliases, _getStoreInstance, getBackendType } from './services/stores/interface.js';
@@ -20,6 +23,7 @@ import { initClientResolver } from './services/client-resolver.js';
2023
import { initLLM } from './services/llm/interface.js';
2124
import { runConsolidation } from './services/consolidation.js';
2225
import { loadAliasCache } from './services/entities.js';
26+
import { runFeedbackLoop } from './services/feedback-loop.js';
2327

2428
process.on('unhandledRejection', (reason) => {
2529
console.error('[unhandled-rejection]', reason);
@@ -49,6 +53,9 @@ app.get('/health', (req, res) => {
4953
res.json({ status: 'ok', service: 'shared-brain', timestamp: new Date().toISOString() });
5054
});
5155

56+
// Dashboard (no auth — it's HTML, API calls use x-api-key header from JS)
57+
app.use('/dashboard', dashboardRouter);
58+
5259
// All other routes require API key + rate limiting
5360
app.use(authMiddleware);
5461
app.use(rateLimitMiddleware);
@@ -63,6 +70,8 @@ app.use('/client', clientRouter);
6370
app.use('/export', exportRouter);
6471
app.use('/graph', graphRouter);
6572
app.use('/reflect', reflectRouter);
73+
app.use('/subscribe', subscribeRouter);
74+
app.use('/collections', collectionsRouter);
6675

6776
async function start() {
6877
try {
@@ -109,6 +118,12 @@ async function start() {
109118
} catch (err) {
110119
console.error('[consolidation] Scheduled run failed:', err.message);
111120
}
121+
// Run feedback loop after consolidation (source trust + stale memory deprioritization)
122+
try {
123+
await runFeedbackLoop();
124+
} catch (err) {
125+
console.error('[feedback-loop] Scheduled run failed:', err.message);
126+
}
112127
});
113128
console.log(`[shared-brain] Consolidation scheduled: ${interval}`);
114129
} catch (llmErr) {

‎api/src/routes/collections.js‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { Router } from 'express';
2+
import { createQdrantCollection, deleteQdrantCollection, listQdrantCollections, getCollectionInfo } from '../services/qdrant.js';
3+
import {
4+
resolveCollection, validateCollectionSlug, registerCollection,
5+
unregisterCollection, listCollections, getDefaultCollection,
6+
} from '../services/collection-registry.js';
7+
8+
export const collectionsRouter = Router();
9+
10+
// GET /collections — List all known collections
11+
collectionsRouter.get('/', async (req, res) => {
12+
try {
13+
// Get registry + actual Qdrant collections
14+
const [registry, qdrantCollections] = await Promise.all([
15+
Promise.resolve(listCollections()),
16+
listQdrantCollections(),
17+
]);
18+
19+
const qdrantNames = new Set(qdrantCollections.map(c => c.name));
20+
21+
// Merge: mark registry entries with Qdrant existence
22+
const merged = registry.map(c => ({
23+
...c,
24+
exists_in_qdrant: qdrantNames.has(c.name),
25+
}));
26+
27+
// Add Qdrant collections not in registry (discovered)
28+
for (const qc of qdrantCollections) {
29+
if (!registry.some(r => r.name === qc.name)) {
30+
merged.push({ name: qc.name, is_default: false, exists_in_qdrant: true, discovered: true });
31+
}
32+
}
33+
34+
res.json({ collections: merged });
35+
} catch (err) {
36+
console.error('[collections:list]', err.message);
37+
res.status(500).json({ error: 'Internal server error' });
38+
}
39+
});
40+
41+
// POST /collections — Create a new collection
42+
collectionsRouter.post('/', async (req, res) => {
43+
try {
44+
const { name, description } = req.body;
45+
46+
const error = validateCollectionSlug(name);
47+
if (error) return res.status(400).json({ error });
48+
49+
const collectionName = resolveCollection(name);
50+
51+
// Check if already exists in Qdrant
52+
try {
53+
await getCollectionInfo(name);
54+
return res.status(409).json({ error: `Collection '${collectionName}' already exists` });
55+
} catch (e) {
56+
if (!e.message?.includes('404')) throw e;
57+
// 404 = doesn't exist, good to create
58+
}
59+
60+
const result = await createQdrantCollection(collectionName);
61+
registerCollection(collectionName, { description: description || '' });
62+
63+
console.log(`[collections] Created: ${collectionName} (${result.dimensions} dims)`);
64+
65+
res.status(201).json({
66+
name: collectionName,
67+
dimensions: result.dimensions,
68+
description: description || '',
69+
created_at: new Date().toISOString(),
70+
});
71+
} catch (err) {
72+
console.error('[collections:create]', err.message);
73+
res.status(500).json({ error: 'Internal server error' });
74+
}
75+
});
76+
77+
// GET /collections/:name — Get collection info
78+
collectionsRouter.get('/:name', async (req, res) => {
79+
try {
80+
const collectionName = resolveCollection(req.params.name);
81+
const info = await getCollectionInfo(req.params.name);
82+
res.json({ name: collectionName, ...info });
83+
} catch (err) {
84+
if (err.message?.includes('404')) {
85+
return res.status(404).json({ error: 'Collection not found' });
86+
}
87+
console.error('[collections:get]', err.message);
88+
res.status(500).json({ error: 'Internal server error' });
89+
}
90+
});
91+
92+
// DELETE /collections/:name — Delete a collection (admin only, not the default)
93+
collectionsRouter.delete('/:name', async (req, res) => {
94+
try {
95+
const collectionName = resolveCollection(req.params.name);
96+
97+
if (collectionName === getDefaultCollection()) {
98+
return res.status(400).json({ error: 'Cannot delete the default collection' });
99+
}
100+
101+
// Agent-scoped keys cannot delete collections
102+
if (req.authenticatedAgent) {
103+
return res.status(403).json({ error: 'Only admin keys can delete collections' });
104+
}
105+
106+
await deleteQdrantCollection(collectionName);
107+
unregisterCollection(collectionName);
108+
109+
console.log(`[collections] Deleted: ${collectionName}`);
110+
111+
res.json({ deleted: true, name: collectionName });
112+
} catch (err) {
113+
if (err.message?.includes('404')) {
114+
return res.status(404).json({ error: 'Collection not found' });
115+
}
116+
console.error('[collections:delete]', err.message);
117+
res.status(500).json({ error: 'Internal server error' });
118+
}
119+
});

‎api/src/routes/dashboard.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { Router } from 'express';
2+
import { readFileSync } from 'fs';
3+
import { fileURLToPath } from 'url';
4+
import { dirname, join } from 'path';
5+
6+
const __dirname = dirname(fileURLToPath(import.meta.url));
7+
const dashboardHtml = readFileSync(join(__dirname, '../templates/dashboard.html'), 'utf-8');
8+
9+
export const dashboardRouter = Router();
10+
11+
// GET /dashboard — Serve dashboard (no auth — it's just HTML, API calls use x-api-key header)
12+
dashboardRouter.get('/', (req, res) => {
13+
res.setHeader('Content-Type', 'text/html; charset=utf-8');
14+
res.send(dashboardHtml);
15+
});

‎api/src/routes/entities.js‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { Router } from 'express';
22
import {
33
isEntityStoreAvailable, listEntities, findEntity, getEntityMemories, getEntityStats,
4+
_getStoreInstance,
45
} from '../services/stores/interface.js';
6+
import { reclassifyEntity } from '../services/entities.js';
7+
import { batchUpdateEntityType } from '../services/qdrant.js';
8+
import { findMisclassifiedEntities } from '../services/entity-type-heuristics.js';
59

610
export const entitiesRouter = Router();
711

@@ -41,6 +45,143 @@ entitiesRouter.get('/stats', async (req, res) => {
4145
}
4246
});
4347

48+
// GET /entities/reclassify/suggestions — Auto-suggest misclassified entities
49+
entitiesRouter.get('/reclassify/suggestions', async (req, res) => {
50+
try {
51+
if (!isEntityStoreAvailable()) {
52+
return res.status(400).json({ error: 'Entity queries require sqlite or postgres backend.' });
53+
}
54+
55+
// Fetch all entities (high limit to scan them all)
56+
const result = await listEntities({ limit: 5000 });
57+
const suggestions = findMisclassifiedEntities(result.results);
58+
59+
res.json({
60+
count: suggestions.length,
61+
suggestions,
62+
});
63+
} catch (err) {
64+
console.error('[entities:reclassify:suggestions]', err.message);
65+
res.status(500).json({ error: 'Internal server error' });
66+
}
67+
});
68+
69+
// POST /entities/reclassify — Reclassify entity types
70+
entitiesRouter.post('/reclassify', async (req, res) => {
71+
try {
72+
if (!isEntityStoreAvailable()) {
73+
return res.status(400).json({ error: 'Entity queries require sqlite or postgres backend.' });
74+
}
75+
76+
const { reclassifications, dry_run } = req.body;
77+
const isDryRun = dry_run !== false; // default true
78+
79+
if (!Array.isArray(reclassifications) || reclassifications.length === 0) {
80+
return res.status(400).json({ error: 'reclassifications array is required and must not be empty' });
81+
}
82+
83+
const VALID_TYPES = ['client', 'person', 'system', 'service', 'domain', 'technology', 'workflow', 'agent'];
84+
85+
// Validate all entries
86+
for (const entry of reclassifications) {
87+
if (!entry.name || typeof entry.name !== 'string') {
88+
return res.status(400).json({ error: `Each reclassification must have a "name" string` });
89+
}
90+
if (!entry.new_type || !VALID_TYPES.includes(entry.new_type)) {
91+
return res.status(400).json({ error: `Invalid new_type "${entry.new_type}" for "${entry.name}". Valid types: ${VALID_TYPES.join(', ')}` });
92+
}
93+
}
94+
95+
const results = [];
96+
97+
for (const entry of reclassifications) {
98+
const entity = await findEntity(entry.name);
99+
if (!entity) {
100+
results.push({
101+
name: entry.name,
102+
old_type: entry.current_type || 'unknown',
103+
new_type: entry.new_type,
104+
memories_affected: 0,
105+
error: 'Entity not found',
106+
});
107+
continue;
108+
}
109+
110+
const oldType = entity.entity_type;
111+
112+
if (isDryRun) {
113+
// Count linked memories for preview
114+
const store = _getStoreInstance();
115+
const linkCount = store?.db
116+
? store.db.prepare('SELECT COUNT(*) as count FROM entity_memory_links WHERE entity_id = @id').get({ id: entity.id })
117+
: { count: 0 };
118+
119+
results.push({
120+
name: entity.canonical_name,
121+
old_type: oldType,
122+
new_type: entry.new_type,
123+
memories_affected: linkCount?.count || 0,
124+
});
125+
} else {
126+
// 1. Update structured store
127+
const storeResult = await reclassifyEntity(entry.name, entry.new_type, {
128+
findEntity,
129+
_getStoreInstance,
130+
});
131+
132+
// 2. Update Qdrant payloads in chunks
133+
let qdrantResult = { total_updated: 0, total_scanned: 0 };
134+
try {
135+
qdrantResult = await batchUpdateEntityType(entity.canonical_name, oldType, entry.new_type);
136+
} catch (err) {
137+
console.error(`[entities:reclassify] Qdrant update failed for "${entry.name}":`, err.message);
138+
}
139+
140+
results.push({
141+
name: entity.canonical_name,
142+
old_type: oldType,
143+
new_type: entry.new_type,
144+
memories_affected: storeResult.memories_affected,
145+
qdrant_updated: qdrantResult.total_updated,
146+
qdrant_scanned: qdrantResult.total_scanned,
147+
});
148+
149+
// 3. Log reclassification as an event in the brain (fire-and-forget)
150+
try {
151+
const internalUrl = `http://localhost:${process.env.PORT || 8084}/memory`;
152+
const apiKey = req.headers['x-api-key'];
153+
fetch(internalUrl, {
154+
method: 'POST',
155+
headers: {
156+
'Content-Type': 'application/json',
157+
...(apiKey ? { 'x-api-key': apiKey } : {}),
158+
},
159+
body: JSON.stringify({
160+
type: 'event',
161+
content: `Entity reclassified: "${entity.canonical_name}" changed from ${oldType} to ${entry.new_type}. ${storeResult.memories_affected} memories linked, ${qdrantResult.total_updated} Qdrant payloads updated.`,
162+
source_agent: 'system',
163+
client_id: 'global',
164+
category: 'episodic',
165+
importance: 'medium',
166+
}),
167+
}).catch(e => console.error('[entities:reclassify:log]', e.message));
168+
} catch (e) {
169+
console.error('[entities:reclassify:log]', e.message);
170+
}
171+
}
172+
}
173+
174+
res.json({
175+
preview: isDryRun ? results : undefined,
176+
applied: isDryRun ? false : results,
177+
dry_run: isDryRun,
178+
});
179+
} catch (err) {
180+
console.error('[entities:reclassify]', err.message);
181+
res.status(500).json({ error: 'Internal server error' });
182+
}
183+
});
184+
44185
// GET /entities/:name — Single entity by name or alias
45186
entitiesRouter.get('/:name', async (req, res) => {
46187
try {

0 commit comments

Comments
 (0)