v1.2.0: Entity extraction, linking, and graph queries

Add automatic entity extraction from memory content at storage time.
Two-speed approach: fast regex path on every write (non-blocking),
LLM refinement during consolidation with alias normalization that
compounds over time. Entities indexed in Qdrant payload for native
filtered vector search with no result-count ceiling.

New: entities service, entity graph API endpoints, brain_entities
MCP tool, backfill script, Qdrant entity index, shared linking
function. Fix scrollPoints dropping false filter values, Postgres
createEntity race condition, MCP tool input validation.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Skiipy11

README.md

@@ -3,6 +3,7 @@ WORKDIR /app
 COPY package*.json ./
 RUN npm install --production
 COPY src/ ./src/
+COPY scripts/ ./scripts/
 RUN mkdir -p /app/data && \
     addgroup --system --gid 1001 nodejs && \
     adduser --system --uid 1001 nodejs && \

api/Dockerfile

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+/**
+ * Backfill entities for existing memories.
+ * Runs fast-path entity extraction (regex + known-tech dictionary) on all active Qdrant points.
+ * No LLM calls — this is purely regex-based extraction.
+ *
+ * Usage: node api/scripts/backfill-entities.js
+ * Requires: .env configured with QDRANT_URL, BRAIN_API_KEY, and STRUCTURED_STORE
+ */
+
+try { await import('dotenv/config'); } catch (e) { /* dotenv not needed in Docker — env injected */ }
+import { initQdrant, scrollPoints, updatePointPayload } from '../src/services/qdrant.js';
+import { initStore, isEntityStoreAvailable, createEntity, findEntity, linkEntityToMemory, loadAllAliases } from '../src/services/stores/interface.js';
+import { extractEntities, loadAliasCache, linkExtractedEntities } from '../src/services/entities.js';
+import { initEmbeddings } from '../src/services/embedders/interface.js';
+
+async function backfill() {
+  console.log('[backfill] Starting entity backfill for existing memories...');
+
+  // Initialize
+  await initEmbeddings();
+  await initQdrant();
+  await initStore();
+
+  if (!isEntityStoreAvailable()) {
+    console.error('[backfill] Entity store not available. Need sqlite or postgres.');
+    process.exit(1);
+  }
+
+  // Load alias cache
+  try {
+    const aliases = await loadAllAliases();
+    loadAliasCache(aliases);
+  } catch (e) {
+    console.log('[backfill] No existing aliases — starting fresh');
+  }
+
+  let processed = 0;
+  let entitiesCreated = 0;
+  let linksCreated = 0;
+  let offset = null;
+
+  while (true) {
+    const result = await scrollPoints({ active: true }, 100, offset);
+    const points = result.points || [];
+
+    if (points.length === 0) break;
+
+    for (const point of points) {
+      const pay = point.payload;
+      const text = pay.text || '';
+      const clientId = pay.client_id || 'global';
+      const sourceAgent = pay.source_agent || 'unknown';
+
+      const entities = extractEntities(text, clientId, sourceAgent);
+
+      if (entities.length > 0) {
+        // Update Qdrant payload with entities
+        const entityPayload = entities.map(e => ({ name: e.name, type: e.type }));
+        await updatePointPayload(point.id, { entities: entityPayload });
+
+        // Create entities and links in structured store
+        await linkExtractedEntities(entities, point.id, { createEntity, findEntity, linkEntityToMemory });
+        entitiesCreated += entities.filter(e => !e.entityId).length;
+        linksCreated += entities.length;
+      }
+
+      processed++;
+      if (processed % 50 === 0) {
+        console.log(`[backfill] Processed ${processed} memories, ${entitiesCreated} entities created, ${linksCreated} links`);
+      }
+    }
+
+    offset = result.next_page_offset;
+    if (!offset) break;
+  }
+
+  console.log(`[backfill] Complete: ${processed} memories processed, ${entitiesCreated} new entities, ${linksCreated} links created`);
+  process.exit(0);
+}
+
+backfill().catch(err => {
+  console.error('[backfill] Fatal:', err);
+  process.exit(1);
+});

api/scripts/backfill-entities.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 /**
  * One-time cleanup script for Shared Brain duplicate memories.
- * Run on Beelink where Qdrant is at localhost:6333.
+ * Run on the server where Qdrant is at localhost:6333.
  *
  * Usage: node cleanup-duplicates.js [--dry-run]
  *

api/scripts/cleanup-duplicates.js

@@ -5,11 +5,13 @@ import { briefingRouter } from './routes/briefing.js';
 import { webhookRouter } from './routes/webhook.js';
 import { statsRouter } from './routes/stats.js';
 import { consolidationRouter } from './routes/consolidation.js';
-import { initQdrant } from './services/qdrant.js';
+import { entitiesRouter } from './routes/entities.js';
+import { initQdrant, ensureEntityIndex } from './services/qdrant.js';
 import { initEmbeddings } from './services/embedders/interface.js';
-import { initStore } from './services/stores/interface.js';
+import { initStore, isEntityStoreAvailable, loadAllAliases } from './services/stores/interface.js';
 import { initLLM } from './services/llm/interface.js';
 import { runConsolidation } from './services/consolidation.js';
+import { loadAliasCache } from './services/entities.js';
 
 // Validate required environment variables
 if (!process.env.BRAIN_API_KEY) {
@@ -36,18 +38,30 @@ app.use('/memory', memoryRouter);
 app.use('/briefing', briefingRouter);
 app.use('/webhook', webhookRouter);
 app.use('/consolidate', consolidationRouter);
+app.use('/entities', entitiesRouter);
 
 async function start() {
   try {
     // Initialize embedding provider first (Qdrant needs dimensions)
     await initEmbeddings();
 
     await initQdrant();
+    await ensureEntityIndex();
     console.log('[shared-brain] Qdrant collection ready');
 
     // Initialize structured storage backend
     await initStore();
 
+    // Load entity alias cache for fast-path extraction
+    if (isEntityStoreAvailable()) {
+      try {
+        const aliases = await loadAllAliases();
+        loadAliasCache(aliases);
+      } catch (e) {
+        console.log('[shared-brain] Entity alias cache: starting empty (first run)');
+      }
+    }
+
     // Initialize consolidation LLM (optional — only if enabled)
     if (process.env.CONSOLIDATION_ENABLED !== 'false') {
       try {

api/src/index.js

@@ -65,6 +65,18 @@ briefingRouter.get('/', async (req, res) => {
     briefing.status_changes.sort(sortDesc);
     briefing.decisions.sort(sortDesc);
 
+    // Collect entities mentioned across all briefing entries
+    const entityCounts = {};
+    for (const point of points) {
+      const entities = point.payload.entities || [];
+      for (const ent of entities) {
+        const key = ent.name;
+        if (!entityCounts[key]) entityCounts[key] = { name: ent.name, type: ent.type, count: 0 };
+        entityCounts[key].count++;
+      }
+    }
+    const entitiesMentioned = Object.values(entityCounts).sort((a, b) => b.count - a.count);
+
     // Summary stats
     briefing.summary = {
       total_entries: points.length,
@@ -74,6 +86,7 @@ briefingRouter.get('/', async (req, res) => {
       decisions: briefing.decisions.length,
       agents_active: [...new Set(points.map(p => p.payload.source_agent))],
       clients_mentioned: [...new Set(points.map(p => p.payload.client_id).filter(c => c !== 'global'))],
+      entities_mentioned: entitiesMentioned.slice(0, 20),
     };
 
     // Get collection stats

api/src/routes/briefing.js

@@ -0,0 +1,96 @@
+import { Router } from 'express';
+import {
+  isEntityStoreAvailable, listEntities, findEntity, getEntityMemories, getEntityStats,
+} from '../services/stores/interface.js';
+
+export const entitiesRouter = Router();
+
+// GET /entities — List all entities
+entitiesRouter.get('/', async (req, res) => {
+  try {
+    if (!isEntityStoreAvailable()) {
+      return res.status(400).json({
+        error: 'Entity queries require sqlite or postgres backend. Set STRUCTURED_STORE in .env.',
+      });
+    }
+
+    const { type: entityType, limit, offset } = req.query;
+    const result = await listEntities({ entity_type: entityType, limit, offset });
+
+    res.json({
+      count: result.results.length,
+      entities: result.results,
+    });
+  } catch (err) {
+    console.error('[entities] Error:', err.message);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+// GET /entities/stats — Entity stats
+entitiesRouter.get('/stats', async (req, res) => {
+  try {
+    if (!isEntityStoreAvailable()) {
+      return res.json({ total: 0, by_type: {}, top_mentioned: [] });
+    }
+    const stats = await getEntityStats();
+    res.json(stats);
+  } catch (err) {
+    console.error('[entities:stats] Error:', err.message);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+// GET /entities/:name — Single entity by name or alias
+entitiesRouter.get('/:name', async (req, res) => {
+  try {
+    if (!isEntityStoreAvailable()) {
+      return res.status(400).json({ error: 'Entity queries require sqlite or postgres backend.' });
+    }
+
+    const entity = await findEntity(req.params.name);
+    if (!entity) {
+      return res.status(404).json({ error: 'Entity not found' });
+    }
+
+    res.json({
+      id: entity.id,
+      canonical_name: entity.canonical_name,
+      entity_type: entity.entity_type,
+      first_seen: entity.first_seen,
+      last_seen: entity.last_seen,
+      mention_count: entity.mention_count,
+      aliases: entity.aliases || [],
+    });
+  } catch (err) {
+    console.error('[entities:get] Error:', err.message);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+// GET /entities/:name/memories — All memories linked to an entity
+entitiesRouter.get('/:name/memories', async (req, res) => {
+  try {
+    if (!isEntityStoreAvailable()) {
+      return res.status(400).json({ error: 'Entity queries require sqlite or postgres backend.' });
+    }
+
+    const entity = await findEntity(req.params.name);
+    if (!entity) {
+      return res.status(404).json({ error: 'Entity not found' });
+    }
+
+    const limit = parseInt(req.query.limit) || 20;
+    const links = await getEntityMemories(entity.id, limit);
+
+    res.json({
+      entity: entity.canonical_name,
+      entity_type: entity.entity_type,
+      count: links.results.length,
+      memory_links: links.results,
+    });
+  } catch (err) {
+    console.error('[entities:memories] Error:', err.message);
+    res.status(500).json({ error: err.message });
+  }
+});

api/src/routes/entities.js

@@ -7,8 +7,10 @@ import {
 } from '../services/qdrant.js';
 import {
   createEvent, upsertFact, upsertStatus, listEvents, listFacts, listStatuses, isStoreAvailable,
+  isEntityStoreAvailable, createEntity, findEntity, linkEntityToMemory,
 } from '../services/stores/interface.js';
 import { scrubCredentials } from '../services/scrub.js';
+import { extractEntities, linkExtractedEntities } from '../services/entities.js';
 
 export const memoryRouter = Router();
 
@@ -104,10 +106,32 @@ memoryRouter.post('/', async (req, res) => {
       ...(metadata ? { metadata } : {}),
     };
 
+    // Extract entities (fast path — regex + alias cache, no LLM)
+    let extractedEntities = [];
+    try {
+      extractedEntities = extractEntities(cleanContent, client_id || 'global', source_agent);
+      if (extractedEntities.length > 0) {
+        payload.entities = extractedEntities.map(e => ({ name: e.name, type: e.type }));
+      }
+    } catch (e) {
+      console.error('[memory:entities] Extraction failed (non-blocking):', e.message);
+    }
+
     // Embed and store in Qdrant
     const vector = await embed(cleanContent);
     await upsertPoint(pointId, vector, payload);
 
+    // Link entities in structured store (fire-and-forget — don't block response)
+    if (isEntityStoreAvailable() && extractedEntities.length > 0) {
+      Promise.resolve().then(async () => {
+        try {
+          await linkExtractedEntities(extractedEntities, pointId, { createEntity, findEntity, linkEntityToMemory });
+        } catch (e) {
+          console.error('[memory:entities] Linking failed:', e.message);
+        }
+      });
+    }
+
     // Store in structured database (if configured)
     const storeData = {
       content: cleanContent,
@@ -160,7 +184,7 @@ memoryRouter.post('/', async (req, res) => {
 // GET /memory/search — Semantic search via Qdrant
 memoryRouter.get('/search', async (req, res) => {
   try {
-    const { q, type, source_agent, client_id, category, limit, include_superseded } = req.query;
+    const { q, type, source_agent, client_id, category, limit, include_superseded, entity } = req.query;
 
     if (!q) {
       return res.status(400).json({ error: 'Missing required query parameter: q' });
@@ -176,7 +200,20 @@ memoryRouter.get('/search', async (req, res) => {
     // By default, only return active memories (not superseded)
     if (include_superseded !== 'true') filter.active = true;
 
-    const rawResults = await searchPoints(vector, filter, parseInt(limit) || 10);
+    // Entity filter — resolve alias to canonical name, then filter via Qdrant payload
+    const nestedFilters = [];
+    if (entity) {
+      let entityName = entity;
+      if (isEntityStoreAvailable()) {
+        try {
+          const found = await findEntity(entity);
+          if (found) entityName = found.canonical_name;
+        } catch (e) { /* use original name */ }
+      }
+      nestedFilters.push({ arrayField: 'entities', key: 'name', value: entityName });
+    }
+
+    const rawResults = await searchPoints(vector, filter, parseInt(limit) || 10, nestedFilters);
 
     // Apply confidence decay and re-rank
     const results = rawResults.map(r => {

api/src/routes/memory.js

@@ -1,5 +1,6 @@
 import { Router } from 'express';
 import { getMemoryStats, scrollPoints, computeEffectiveConfidence, DECAY_TYPES } from '../services/qdrant.js';
+import { isEntityStoreAvailable, getEntityStats } from '../services/stores/interface.js';
 
 export const statsRouter = Router();
 
@@ -21,13 +22,22 @@ statsRouter.get('/', async (req, res) => {
       // Non-critical
     }
 
+    // Entity stats
+    let entityStats = null;
+    if (isEntityStoreAvailable()) {
+      try {
+        entityStats = await getEntityStats();
+      } catch (e) { /* non-critical */ }
+    }
+
     res.json({
       ...stats,
       decayed_below_50pct: decayedCount,
       decay_config: {
         factor: parseFloat(process.env.DECAY_FACTOR) || 0.98,
         affected_types: DECAY_TYPES,
       },
+      ...(entityStats ? { entities: entityStats } : {}),
     });
   } catch (err) {
     console.error('[stats] Error:', err.message);