security: input validation, rate limiting, and consolidation hardening

Fixes identified during security audit triggered by #1:

Input validation (middleware/validate.js):
- source_agent must match /^[a-zA-Z0-9_-]{1,64}$/
- importance restricted to allowlist (critical/high/medium/low)
- content capped at 10,000 characters
- metadata validated for depth (max 3) and size (max 10KB)
- client_id, key, subject, status_value length-capped

Rate limiting (middleware/ratelimit.js):
- 60 writes/min, 120 reads/min, 1 consolidation/hour per API key
- Configurable via RATE_LIMIT_WRITES, RATE_LIMIT_READS env vars
- Automatic stale bucket cleanup

Corroboration hardening:
- observed_by array capped at 20 entries to prevent bloat attacks
- Search limit capped at 100 results max

Credential scrubbing improvements (scrub.js):
- Added AWS access key pattern (AKIA...)
- Added connection string pattern (postgres://, mongodb://, redis://)
- Added OpenAI/Anthropic API key pattern (sk-proj-..., sk-ant-...)
- New scrubObject() recursively scrubs metadata before storage

Consolidation LLM injection defense (consolidation.js):
- Memory content wrapped in XML tags with escaped angle brackets
- All memory IDs in LLM output validated against current batch
- Contradictions with out-of-batch IDs filtered out
- Connection/entity mentioned_in IDs validated
- LLM-provided importance values sanitized against allowlist

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Skiipy11

api/src/index.js

@@ -1,5 +1,6 @@
 import express from 'express';
 import { authMiddleware } from './middleware/auth.js';
+import { rateLimitMiddleware } from './middleware/ratelimit.js';
 import { memoryRouter } from './routes/memory.js';
 import { briefingRouter } from './routes/briefing.js';
 import { webhookRouter } from './routes/webhook.js';
@@ -30,8 +31,9 @@ app.get('/health', (req, res) => {
   res.json({ status: 'ok', service: 'shared-brain', timestamp: new Date().toISOString() });
 });
 
-// All other routes require API key
+// All other routes require API key + rate limiting
 app.use(authMiddleware);
+app.use(rateLimitMiddleware);
 
 app.use('/stats', statsRouter);
 app.use('/memory', memoryRouter);

api/src/middleware/ratelimit.js

@@ -0,0 +1,69 @@
+// Per-key request rate limiting for authenticated endpoints
+
+const buckets = new Map(); // key -> { writes: { count, windowStart }, reads: { ... }, consolidation: { ... } }
+
+const LIMITS = {
+  write: { max: parseInt(process.env.RATE_LIMIT_WRITES) || 60, windowMs: 60_000 },
+  read: { max: parseInt(process.env.RATE_LIMIT_READS) || 120, windowMs: 60_000 },
+  consolidation: { max: 1, windowMs: 3_600_000 }, // 1 per hour
+};
+
+function getBucket(apiKey, type) {
+  const now = Date.now();
+  if (!buckets.has(apiKey)) buckets.set(apiKey, {});
+  const keyBuckets = buckets.get(apiKey);
+
+  if (!keyBuckets[type] || now - keyBuckets[type].windowStart > LIMITS[type].windowMs) {
+    keyBuckets[type] = { count: 0, windowStart: now };
+  }
+
+  return keyBuckets[type];
+}
+
+function checkLimit(apiKey, type) {
+  const bucket = getBucket(apiKey, type);
+  if (bucket.count >= LIMITS[type].max) {
+    const retryAfter = Math.ceil((bucket.windowStart + LIMITS[type].windowMs - Date.now()) / 1000);
+    return { limited: true, retryAfter };
+  }
+  bucket.count++;
+  return { limited: false };
+}
+
+// Classify route + method into a rate limit type
+function classifyRequest(method, path) {
+  if (path.startsWith('/consolidate') && method === 'POST') return 'consolidation';
+  if (method === 'POST' || method === 'PUT' || method === 'PATCH' || method === 'DELETE') return 'write';
+  return 'read';
+}
+
+export function rateLimitMiddleware(req, res, next) {
+  const apiKey = req.headers['x-api-key'] || 'unknown';
+  const type = classifyRequest(req.method, req.path);
+  const { limited, retryAfter } = checkLimit(apiKey, type);
+
+  if (limited) {
+    res.set('Retry-After', String(retryAfter));
+    return res.status(429).json({
+      error: `Rate limit exceeded for ${type} requests. Try again in ${retryAfter}s.`,
+      limit: LIMITS[type].max,
+      window_seconds: LIMITS[type].windowMs / 1000,
+    });
+  }
+
+  next();
+}
+
+// Periodic cleanup of stale buckets (every 10 minutes)
+setInterval(() => {
+  const now = Date.now();
+  for (const [key, keyBuckets] of buckets) {
+    let allExpired = true;
+    for (const [type, bucket] of Object.entries(keyBuckets)) {
+      if (now - bucket.windowStart <= LIMITS[type]?.windowMs) {
+        allExpired = false;
+      }
+    }
+    if (allExpired) buckets.delete(key);
+  }
+}, 600_000).unref();

api/src/middleware/validate.js

@@ -0,0 +1,79 @@
+// Input validation middleware and helpers for memory API
+
+const AGENT_NAME_REGEX = /^[a-zA-Z0-9_-]{1,64}$/;
+const VALID_TYPES = ['event', 'fact', 'decision', 'status'];
+const VALID_IMPORTANCE = ['critical', 'high', 'medium', 'low'];
+const MAX_CONTENT_LENGTH = 10_000;
+const MAX_METADATA_SIZE = 10_240; // 10 KB serialized
+const MAX_METADATA_DEPTH = 3;
+const MAX_OBSERVED_BY = 20;
+const MAX_STRING_FIELD_LENGTH = 256;
+
+function checkDepth(obj, max, current = 0) {
+  if (current >= max) return false;
+  if (obj === null || typeof obj !== 'object') return true;
+  if (Array.isArray(obj)) {
+    return obj.every(item => checkDepth(item, max, current + 1));
+  }
+  return Object.values(obj).every(val => checkDepth(val, max, current + 1));
+}
+
+export function validateSourceAgent(agent) {
+  if (!agent || typeof agent !== 'string') return 'source_agent is required and must be a string';
+  if (!AGENT_NAME_REGEX.test(agent)) return `source_agent must match ${AGENT_NAME_REGEX} (1-64 alphanumeric, hyphens, underscores)`;
+  return null;
+}
+
+export function validateType(type) {
+  if (!type) return 'type is required';
+  if (!VALID_TYPES.includes(type)) return `Invalid type: ${type}. Must be one of: ${VALID_TYPES.join(', ')}`;
+  return null;
+}
+
+export function validateImportance(importance) {
+  if (!importance) return null; // optional, defaults to 'medium'
+  if (!VALID_IMPORTANCE.includes(importance)) return `Invalid importance: ${importance}. Must be one of: ${VALID_IMPORTANCE.join(', ')}`;
+  return null;
+}
+
+export function validateContent(content) {
+  if (!content || typeof content !== 'string') return 'content is required and must be a string';
+  if (content.length > MAX_CONTENT_LENGTH) return `content exceeds maximum length of ${MAX_CONTENT_LENGTH} characters (got ${content.length})`;
+  return null;
+}
+
+export function validateMetadata(metadata) {
+  if (metadata === undefined || metadata === null) return null; // optional
+  if (typeof metadata !== 'object' || Array.isArray(metadata)) return 'metadata must be a plain object';
+  const serialized = JSON.stringify(metadata);
+  if (serialized.length > MAX_METADATA_SIZE) return `metadata exceeds maximum size of ${MAX_METADATA_SIZE} bytes (got ${serialized.length})`;
+  if (!checkDepth(metadata, MAX_METADATA_DEPTH)) return `metadata exceeds maximum nesting depth of ${MAX_METADATA_DEPTH}`;
+  return null;
+}
+
+export function validateStringField(value, name, maxLen = MAX_STRING_FIELD_LENGTH) {
+  if (value === undefined || value === null) return null; // optional
+  if (typeof value !== 'string') return `${name} must be a string`;
+  if (value.length > maxLen) return `${name} exceeds maximum length of ${maxLen} characters`;
+  return null;
+}
+
+export function validateClientId(clientId) {
+  return validateStringField(clientId, 'client_id', 64);
+}
+
+// Validate all inputs for POST /memory and return first error or null
+export function validateMemoryInput({ type, content, source_agent, importance, metadata, client_id, key, subject, status_value }) {
+  return validateType(type)
+    || validateContent(content)
+    || validateSourceAgent(source_agent)
+    || validateImportance(importance)
+    || validateMetadata(metadata)
+    || validateClientId(client_id)
+    || validateStringField(key, 'key', 128)
+    || validateStringField(subject, 'subject', 256)
+    || validateStringField(status_value, 'status_value', 256)
+    || null;
+}
+
+export { MAX_OBSERVED_BY, VALID_TYPES, VALID_IMPORTANCE };

api/src/routes/memory.js

@@ -9,8 +9,9 @@ import {
   createEvent, upsertFact, upsertStatus, listEvents, listFacts, listStatuses, isStoreAvailable,
   isEntityStoreAvailable, createEntity, findEntity, linkEntityToMemory,
 } from '../services/stores/interface.js';
-import { scrubCredentials } from '../services/scrub.js';
+import { scrubCredentials, scrubObject } from '../services/scrub.js';
 import { extractEntities, linkExtractedEntities } from '../services/entities.js';
+import { validateMemoryInput, MAX_OBSERVED_BY } from '../middleware/validate.js';
 
 export const memoryRouter = Router();
 
@@ -19,16 +20,10 @@ memoryRouter.post('/', async (req, res) => {
   try {
     const { type, content, source_agent, client_id, category, importance, metadata } = req.body;
 
-    // Validate required fields
-    if (!type || !content || !source_agent) {
-      return res.status(400).json({
-        error: 'Missing required fields: type, content, source_agent',
-        valid_types: ['event', 'fact', 'decision', 'status'],
-      });
-    }
-
-    if (!['event', 'fact', 'decision', 'status'].includes(type)) {
-      return res.status(400).json({ error: `Invalid type: ${type}. Must be event, fact, decision, or status` });
+    // Validate all input fields
+    const validationError = validateMemoryInput(req.body);
+    if (validationError) {
+      return res.status(400).json({ error: validationError });
     }
 
     // Scrub credentials
@@ -58,6 +53,18 @@ memoryRouter.post('/', async (req, res) => {
       }
 
       // Different agent → corroborate: record that another agent observed the same thing
+      if (existingObservedBy.length >= MAX_OBSERVED_BY) {
+        return res.status(200).json({
+          id: existing.id,
+          type: existing.payload.type,
+          content_hash: contentHash,
+          deduplicated: true,
+          observed_by: existingObservedBy,
+          observation_count: existingObservedBy.length,
+          message: `Observer cap reached (${MAX_OBSERVED_BY}) — corroboration noted but not recorded`,
+          stored_in: { qdrant: true, structured_db: true },
+        });
+      }
       const updatedObservedBy = [...existingObservedBy, source_agent];
       const now = new Date().toISOString();
       await updatePointPayload(existing.id, {
@@ -132,7 +139,7 @@ memoryRouter.post('/', async (req, res) => {
       superseded_by: null,
       ...(type === 'fact' && req.body.key ? { key: req.body.key } : {}),
       ...(type === 'status' && req.body.subject ? { subject: req.body.subject, status_value: req.body.status_value } : {}),
-      ...(metadata ? { metadata } : {}),
+      ...(metadata ? { metadata: scrubObject(metadata) } : {}),
     };
 
     // Extract entities (fast path — regex + alias cache, no LLM)
@@ -242,7 +249,7 @@ memoryRouter.get('/search', async (req, res) => {
       nestedFilters.push({ arrayField: 'entities', key: 'name', value: entityName });
     }
 
-    const rawResults = await searchPoints(vector, filter, parseInt(limit) || 10, nestedFilters);
+    const rawResults = await searchPoints(vector, filter, Math.min(parseInt(limit) || 10, 100), nestedFilters);
 
     // Apply confidence decay and re-rank
     const results = rawResults.map(r => {

api/src/routes/webhook.js

@@ -5,6 +5,7 @@ import { upsertPoint, findByPayload, updatePointPayload } from '../services/qdra
 import { createEvent, upsertStatus, isStoreAvailable, isEntityStoreAvailable, createEntity, findEntity, linkEntityToMemory } from '../services/stores/interface.js';
 import { scrubCredentials } from '../services/scrub.js';
 import { extractEntities, linkExtractedEntities } from '../services/entities.js';
+import { validateClientId, MAX_OBSERVED_BY } from '../middleware/validate.js';
 
 export const webhookRouter = Router();
 
@@ -37,6 +38,9 @@ webhookRouter.post('/n8n', async (req, res) => {
       });
     }
 
+    const clientIdError = validateClientId(client_id);
+    if (clientIdError) return res.status(400).json({ error: clientIdError });
+
     const now = new Date().toISOString();
 
     // Build content string
@@ -67,7 +71,16 @@ webhookRouter.post('/n8n', async (req, res) => {
         });
       }
 
-      // Different source → corroborate
+      // Different source → corroborate (with cap)
+      if (existingObservedBy.length >= MAX_OBSERVED_BY) {
+        return res.status(200).json({
+          id: existing.id,
+          deduplicated: true,
+          observed_by: existingObservedBy,
+          observation_count: existingObservedBy.length,
+          message: `Observer cap reached (${MAX_OBSERVED_BY}) — corroboration noted but not recorded`,
+        });
+      }
       const updatedObservedBy = [...existingObservedBy, sourceAgent];
       await updatePointPayload(existing.id, {
         observed_by: updatedObservedBy,

api/src/services/consolidation.js

@@ -183,11 +183,16 @@ export async function runConsolidation() {
 }
 
 async function consolidateBatch(points, clientId) {
-  // Format memories for the LLM
+  // Collect valid IDs for output validation
+  const batchIds = new Set(points.map(p => p.id));
+
+  // Format memories for the LLM — wrapped in XML tags to resist prompt injection
   const memoriesText = points.map(p => {
     const pay = p.payload;
-    return `[ID: ${p.id}] [Type: ${pay.type}] [Agent: ${pay.source_agent}] [Client: ${pay.client_id}] [Created: ${pay.created_at}]\n${pay.text}`;
-  }).join('\n\n---\n\n');
+    // Escape any XML-like tags in the memory content to prevent tag injection
+    const safeText = pay.text.replace(/</g, '&lt;').replace(/>/g, '&gt;');
+    return `<memory id="${p.id}" type="${pay.type}" agent="${pay.source_agent}" client="${pay.client_id}" created="${pay.created_at}">\n${safeText}\n</memory>`;
+  }).join('\n\n');
 
   const prompt = CONSOLIDATION_PROMPT + memoriesText;
   const responseText = await complete(prompt);
@@ -200,6 +205,45 @@ async function consolidateBatch(points, clientId) {
     return { merged: 0, contradictions: 0, connections: 0, insights: 0 };
   }
 
+  // Validate: strip any memory IDs not in the current batch
+  if (result.merged_facts) {
+    for (const fact of result.merged_facts) {
+      if (fact.source_memories) {
+        fact.source_memories = fact.source_memories.filter(id => batchIds.has(id));
+      }
+    }
+  }
+  if (result.contradictions) {
+    result.contradictions = result.contradictions.filter(c =>
+      batchIds.has(c.memory_a) && batchIds.has(c.memory_b)
+    );
+  }
+  if (result.connections) {
+    for (const conn of result.connections) {
+      if (conn.memories) {
+        conn.memories = conn.memories.filter(id => batchIds.has(id));
+      }
+    }
+    result.connections = result.connections.filter(c => c.memories && c.memories.length >= 2);
+  }
+  if (result.insights) {
+    for (const insight of result.insights) {
+      if (insight.source_memories) {
+        insight.source_memories = insight.source_memories.filter(id => batchIds.has(id));
+      }
+    }
+  }
+  if (result.entities) {
+    for (const ent of result.entities) {
+      if (ent.mentioned_in) {
+        ent.mentioned_in = ent.mentioned_in.filter(id => batchIds.has(id));
+      }
+    }
+  }
+
+  const VALID_IMPORTANCE = ['critical', 'high', 'medium', 'low'];
+  const sanitizeImportance = (val) => VALID_IMPORTANCE.includes(val) ? val : 'medium';
+
   const now = new Date().toISOString();
   let merged = 0, contradictions = 0, connections = 0, insights = 0;
 
@@ -232,7 +276,7 @@ async function consolidateBatch(points, clientId) {
         source_agent: 'consolidation-engine',
         client_id: fact.client_id || clientId,
         category: 'semantic',
-        importance: fact.importance || 'medium',
+        importance: sanitizeImportance(fact.importance),
         key: fact.key || contentHash,
         content_hash: contentHash,
         created_at: now,
@@ -321,7 +365,7 @@ async function consolidateBatch(points, clientId) {
         source_agent: 'consolidation-engine',
         client_id: clientId,
         category: 'semantic',
-        importance: insight.importance || 'medium',
+        importance: sanitizeImportance(insight.importance),
         content_hash: contentHash,
         created_at: now,
         last_accessed_at: now,

api/src/services/scrub.js

@@ -11,6 +11,12 @@ const PATTERNS = [
   { regex: /(?:smtp|email|mail).*?(?:pass|password)\s*[:=]\s*['"]?[^\s'"]{8,}['"]?/gi, replace: '[EMAIL_CRED_REDACTED]' },
   // SSH private keys
   { regex: /-----BEGIN [\w\s]+ PRIVATE KEY-----[\s\S]*?-----END [\w\s]+ PRIVATE KEY-----/g, replace: '[PRIVATE_KEY_REDACTED]' },
+  // AWS access key IDs
+  { regex: /AKIA[0-9A-Z]{16}/g, replace: '[AWS_KEY_REDACTED]' },
+  // Connection strings (postgres, mongodb, redis, mysql)
+  { regex: /(?:postgres(?:ql)?|mongodb(?:\+srv)?|redis|mysql|amqp):\/\/[^\s'"]+/gi, replace: '[CONNECTION_STRING_REDACTED]' },
+  // OpenAI / Anthropic API keys
+  { regex: /sk-(?:proj-|ant-)?[A-Za-z0-9_-]{20,}/g, replace: '[API_KEY_REDACTED]' },
 ];
 
 export function scrubCredentials(text) {
@@ -21,3 +27,18 @@ export function scrubCredentials(text) {
   }
   return scrubbed;
 }
+
+// Recursively scrub all string values in an object
+export function scrubObject(obj) {
+  if (obj === null || obj === undefined) return obj;
+  if (typeof obj === 'string') return scrubCredentials(obj);
+  if (Array.isArray(obj)) return obj.map(item => scrubObject(item));
+  if (typeof obj === 'object') {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = scrubObject(value);
+    }
+    return result;
+  }
+  return obj; // numbers, booleans, etc.
+}