v1.1.0: Consolidation dedup, Gemini Flash, health checks, event TTL

Fix consolidation engine creating duplicate memories on every 6h run:
- Add exact (content_hash) + semantic (92% vector similarity) dedup
  checks before creating merged facts and insights
- Add content_hash dedup to webhook endpoint (n8n events)
- Remove consolidation self-logging (was accumulating as noise)
- Tighten consolidation prompt to reject vague platitudes
- Add Gemini provider (gemini-2.5-flash recommended for consolidation)
- Add EVENT_TTL_DAYS auto-cleanup for old unaccessed events
- Fix Docker health checks (curl not available in containers)
- Include cleanup-duplicates.js script for one-time dedup

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Skiipy11

README.md

@@ -465,9 +465,11 @@ All configuration is via environment variables. Copy `.env.example` to `.env` an
 |----------|---------|-------------|
 | `CONSOLIDATION_ENABLED` | `true` | Enable/disable the consolidation engine |
 | `CONSOLIDATION_INTERVAL` | `0 */6 * * *` | Cron schedule (default: every 6 hours) |
-| `CONSOLIDATION_LLM` | `openai` | `openai`, `anthropic`, or `ollama` |
-| `CONSOLIDATION_MODEL` | `gpt-4o-mini` | Model for consolidation |
+| `CONSOLIDATION_LLM` | `openai` | `openai`, `anthropic`, `gemini`, or `ollama` |
+| `CONSOLIDATION_MODEL` | `gpt-4o-mini` | Model for consolidation (e.g. `gemini-2.5-flash`) |
 | `ANTHROPIC_API_KEY` | — | Required when using Anthropic for consolidation |
+| `GEMINI_API_KEY` | — | Required when using Gemini for consolidation |
+| `EVENT_TTL_DAYS` | `30` | Auto-expire old unaccessed events after this many days |
 
 ### Memory Decay
 

api/scripts/cleanup-duplicates.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+/**
+ * One-time cleanup script for Shared Brain duplicate memories.
+ * Run on Beelink where Qdrant is at localhost:6333.
+ *
+ * Usage: node cleanup-duplicates.js [--dry-run]
+ *
+ * What it removes:
+ * 1. Duplicate content_hash entries (keeps newest, deletes older copies)
+ * 2. Consolidation run log events ("Consolidation run: processed N memories...")
+ * 3. Vague consolidation "insight" platitudes
+ */
+
+const QDRANT_URL = process.env.QDRANT_URL || 'http://localhost:6333';
+const COLLECTION = 'shared_memories';
+const DRY_RUN = process.argv.includes('--dry-run');
+
+const QDRANT_API_KEY = process.env.QDRANT_API_KEY || '';
+
+async function qdrantRequest(path, options = {}) {
+  const headers = { 'Content-Type': 'application/json', ...options.headers };
+  if (QDRANT_API_KEY) headers['api-key'] = QDRANT_API_KEY;
+  const res = await fetch(`${QDRANT_URL}${path}`, {
+    ...options,
+    headers,
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`Qdrant ${path}: ${res.status} ${text}`);
+  }
+  return res.json();
+}
+
+async function scrollAll() {
+  const points = [];
+  let offset = null;
+  while (true) {
+    const body = { limit: 100, with_payload: true, with_vector: false };
+    if (offset) body.offset = offset;
+    const result = await qdrantRequest(`/collections/${COLLECTION}/points/scroll`, {
+      method: 'POST',
+      body: JSON.stringify(body),
+    });
+    points.push(...result.result.points);
+    offset = result.result.next_page_offset;
+    if (!offset) break;
+  }
+  return points;
+}
+
+async function deletePoints(ids) {
+  if (ids.length === 0) return;
+  if (DRY_RUN) {
+    console.log(`  [DRY RUN] Would delete ${ids.length} points`);
+    return;
+  }
+  await qdrantRequest(`/collections/${COLLECTION}/points/delete`, {
+    method: 'POST',
+    body: JSON.stringify({ points: ids }),
+  });
+  console.log(`  Deleted ${ids.length} points`);
+}
+
+// Platitude patterns — vague consolidation insights with no actionable content
+const PLATITUDE_PATTERNS = [
+  /indicates? (?:effective|strong|an ongoing|a robust) (?:project management|integration|collaboration|commitment)/i,
+  /ongoing commitment to improving the efficiency/i,
+  /operational (?:status|capabilities?) (?:of|and) the (?:Shared Brain|system)/i,
+  /closely (?:intertwined|linked|related)/i,
+  /successful completion of workflows indicates/i,
+  /cross-referencing of (?:memory )?repositories indicate/i,
+  /a critical factor in the successful launch/i,
+];
+
+function isPlatitude(text) {
+  return PLATITUDE_PATTERNS.some(p => p.test(text));
+}
+
+async function main() {
+  console.log(`Shared Brain Cleanup ${DRY_RUN ? '(DRY RUN)' : '(LIVE)'}`);
+  console.log('---');
+
+  // Step 1: Scroll all points
+  console.log('Scrolling all points...');
+  const allPoints = await scrollAll();
+  console.log(`Total points: ${allPoints.length}`);
+
+  const toDelete = new Set();
+
+  // Step 2: Find duplicate content_hash entries (keep newest)
+  console.log('\n--- Duplicate content_hash entries ---');
+  const byHash = {};
+  for (const p of allPoints) {
+    const hash = p.payload?.content_hash;
+    if (!hash) continue;
+    if (!byHash[hash]) byHash[hash] = [];
+    byHash[hash].push(p);
+  }
+
+  let dupGroups = 0;
+  let dupCount = 0;
+  for (const [hash, points] of Object.entries(byHash)) {
+    if (points.length <= 1) continue;
+    // Sort by created_at descending — keep newest
+    points.sort((a, b) => new Date(b.payload.created_at) - new Date(a.payload.created_at));
+    const keep = points[0];
+    const dupes = points.slice(1);
+    dupGroups++;
+    dupCount += dupes.length;
+    console.log(`  hash=${hash} (${points.length} copies) — keeping ${keep.id} (${keep.payload.created_at}), deleting ${dupes.length}`);
+    for (const d of dupes) {
+      toDelete.add(d.id);
+    }
+  }
+  console.log(`Found ${dupGroups} duplicate groups, ${dupCount} entries to remove`);
+
+  // Step 3: Find consolidation run log events
+  console.log('\n--- Consolidation run log events ---');
+  let runLogCount = 0;
+  for (const p of allPoints) {
+    if (p.payload?.source_agent === 'consolidation-engine' &&
+        p.payload?.type === 'event' &&
+        p.payload?.text?.startsWith('Consolidation run: processed')) {
+      if (!toDelete.has(p.id)) {
+        toDelete.add(p.id);
+        runLogCount++;
+      }
+    }
+  }
+  console.log(`Found ${runLogCount} consolidation run logs to remove`);
+
+  // Step 4: Find platitude insights
+  console.log('\n--- Platitude insights ---');
+  let platitudeCount = 0;
+  for (const p of allPoints) {
+    if (p.payload?.source_agent === 'consolidation-engine' &&
+        p.payload?.text &&
+        isPlatitude(p.payload.text)) {
+      if (!toDelete.has(p.id)) {
+        toDelete.add(p.id);
+        platitudeCount++;
+        console.log(`  "${p.payload.text.slice(0, 80)}..."`);
+      }
+    }
+  }
+  console.log(`Found ${platitudeCount} platitude insights to remove`);
+
+  // Step 5: Execute deletions
+  const deleteIds = Array.from(toDelete);
+  console.log(`\n--- Total: ${deleteIds.length} points to delete ---`);
+
+  if (deleteIds.length > 0) {
+    // Delete in batches of 100
+    for (let i = 0; i < deleteIds.length; i += 100) {
+      const batch = deleteIds.slice(i, i + 100);
+      await deletePoints(batch);
+    }
+  }
+
+  console.log('\nDone.');
+}
+
+main().catch(err => {
+  console.error('Error:', err.message);
+  process.exit(1);
+});

api/src/routes/webhook.js

@@ -1,7 +1,7 @@
 import { Router } from 'express';
 import crypto from 'crypto';
 import { embed } from '../services/embedders/interface.js';
-import { upsertPoint } from '../services/qdrant.js';
+import { upsertPoint, findByPayload } from '../services/qdrant.js';
 import { createEvent, upsertStatus, isStoreAvailable } from '../services/stores/interface.js';
 import { scrubCredentials } from '../services/scrub.js';
 
@@ -48,6 +48,17 @@ webhookRouter.post('/n8n', async (req, res) => {
     content = scrubCredentials(content);
 
     const contentHash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
+
+    // Dedup: if identical content already exists, return existing memory
+    const duplicates = await findByPayload('content_hash', contentHash, { active: true });
+    if (duplicates.length > 0) {
+      return res.status(200).json({
+        id: duplicates[0].id,
+        deduplicated: true,
+        message: 'Identical webhook event already exists — skipped',
+      });
+    }
+
     const pointId = crypto.randomUUID();
 
     // Store as event in Qdrant