feat: add ask-self and ask-self-orig scaffolding

Author: noelsaw1

ask-self-orig/helpers.js

@@ -0,0 +1,120 @@
+// Pure-function helpers for the RAG module. No I/O, no global state — safe to
+// import from either the CJS query module (src/rag/index.js) or the ESM ingest
+// script (src/rag/ingest.mjs). Every function here is designed to be unit-tested
+// without mocks, network access, or a filesystem.
+
+const CHUNK_TARGET_CHARS = 4800;  // ~1200 tokens
+const CHUNK_OVERLAP_CHARS = 600;  // ~150 tokens
+
+// Priority boosts applied during retrieval re-rank. Higher priority wins at
+// near-equal distance. See the spike learning in PROJECT/2-WORKING/P1-CODE-RAG.md
+// for why strategy/changelog beat regular docs.
+const PRIORITY = {
+  feature_map: 10,
+  strategy: 5,
+  changelog_entry: 5,
+  doc: 1,
+  pr: 1,
+};
+
+/**
+ * Split text into overlapping chunks sized for embedding. Returns the original
+ * text as a single-element array if it's already short enough.
+ *
+ * @param {string} text
+ * @param {{targetChars?: number, overlap?: number}} [options]
+ * @returns {string[]}
+ */
+function chunkText(text, { targetChars = CHUNK_TARGET_CHARS, overlap = CHUNK_OVERLAP_CHARS } = {}) {
+  if (typeof text !== 'string') throw new TypeError('chunkText: text must be a string');
+  if (targetChars <= 0) throw new RangeError('chunkText: targetChars must be > 0');
+  if (overlap < 0 || overlap >= targetChars) throw new RangeError('chunkText: overlap must be >= 0 and < targetChars');
+  if (text.length === 0) return [];
+  if (text.length <= targetChars) return [text];
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const end = Math.min(i + targetChars, text.length);
+    chunks.push(text.slice(i, end));
+    if (end >= text.length) break;
+    i = end - overlap;
+  }
+  return chunks;
+}
+
+/**
+ * Split a CHANGELOG.md into one chunk per version entry. Version headers are
+ * expected to match `## 1.2.3` at the start of a line. The returned version
+ * string is the `1.2.3` capture from the header, or null if the chunk didn't
+ * start with a version header (e.g., a preamble paragraph).
+ *
+ * @param {string} text
+ * @returns {Array<{version: string|null, content: string}>}
+ */
+function chunkChangelog(text) {
+  if (typeof text !== 'string') throw new TypeError('chunkChangelog: text must be a string');
+  const parts = text.split(/(?=^##\s+\d+\.\d+\.\d+)/m).map((s) => s.trim()).filter(Boolean);
+  return parts.map((entry) => {
+    const versionMatch = entry.match(/^##\s+(\d+\.\d+\.\d+)/);
+    return { version: versionMatch ? versionMatch[1] : null, content: entry };
+  });
+}
+
+/**
+ * Classify a doc by its repo-relative path. Strategy/PMF/positioning docs and
+ * the CHANGELOG get a priority boost so retrieval surfaces them for marketing
+ * questions even when semantic distance is close.
+ *
+ * @param {string} relPath - repo-relative path (forward slashes)
+ * @returns {{source: 'changelog'|'strategy'|'doc', priority: number}}
+ */
+function classifyDoc(relPath) {
+  if (typeof relPath !== 'string' || relPath.length === 0) {
+    throw new TypeError('classifyDoc: relPath must be a non-empty string');
+  }
+  if (/changelog\.md$/i.test(relPath)) {
+    return { source: 'changelog', priority: PRIORITY.changelog_entry };
+  }
+  if (/strategy|product.*brief|moat|pmf|positioning/i.test(relPath)) {
+    return { source: 'strategy', priority: PRIORITY.strategy };
+  }
+  return { source: 'doc', priority: PRIORITY.doc };
+}
+
+/**
+ * Format retrieved chunks into a single context string for the synthesis model,
+ * budget-capped by total character count. Each chunk gets a source-aware header
+ * so the model can cite it correctly. Oldest-first order is preserved.
+ *
+ * @param {Array<{source: string, path?: string, pr_number?: number|null, version?: string|null, content: string}>} hits
+ * @param {number} [maxContextChars=80000]
+ * @returns {string}
+ */
+function formatContext(hits, maxContextChars = 80000) {
+  if (!Array.isArray(hits)) throw new TypeError('formatContext: hits must be an array');
+  const parts = [];
+  let totalChars = 0;
+  for (const h of hits) {
+    if (!h || typeof h.content !== 'string') continue;
+    const header = h.source === 'pr'
+      ? `[PR #${h.pr_number}]`
+      : h.source === 'changelog'
+        ? `[changelog.md${h.version ? ` — ${h.version}` : ''}]`
+        : `[${h.path ?? h.source}]`;
+    const block = `=== ${header} ===\n${h.content}\n`;
+    if (totalChars + block.length > maxContextChars) break;
+    parts.push(block);
+    totalChars += block.length;
+  }
+  return parts.join('\n');
+}
+
+module.exports = {
+  CHUNK_TARGET_CHARS,
+  CHUNK_OVERLAP_CHARS,
+  PRIORITY,
+  chunkText,
+  chunkChangelog,
+  classifyDoc,
+  formatContext,
+};

ask-self-orig/index.js

@@ -0,0 +1,171 @@
+// Sleuth Code RAG — query module.
+// Exports askSelf(query, teamId) used by chat-module for the `ask-self` command.
+// Tenancy gate is layer 2 (module-level) per PROJECT/2-WORKING/P1-CODE-RAG.md.
+
+const path = require('node:path');
+const fs = require('node:fs');
+const { formatContext } = require('./helpers.js');
+
+const MODULE_DIR = __dirname;
+const REPO_ROOT = path.join(MODULE_DIR, '..', '..');
+const DB_PATH = path.join(REPO_ROOT, 'data', 'rag', 'sleuth-rag.sqlite');
+const PROMPTS_PATH = path.join(MODULE_DIR, 'prompts.json');
+
+const EMBED_MODEL = 'gemini-embedding-001';
+const EMBED_DIM = 768;
+const SYNTHESIS_MODEL = 'gemini-pro-latest'; // rolling alias — always newest Gemini Pro
+const TOP_K = 20;                 // retrieve generously, trust Gemini to sort
+const PRIORITY_BOOST = 0.02;      // small nudge — doesn't override clear semantic wins
+const MAX_CONTEXT_CHARS = 80000;  // ~20k tokens — spike showed 18k works well
+
+class TenancyError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'TenancyError';
+  }
+}
+
+// Lazy-loaded singletons so a missing env var at boot doesn't kill the process.
+// They throw on first askSelf() call instead, which chat-module catches silently.
+let _db = null;
+let _prompts = null;
+
+function getDb() {
+  if (_db) return _db;
+  if (!fs.existsSync(DB_PATH)) {
+    throw new Error(`RAG index missing at ${DB_PATH}. Run: npm run rag:ingest`);
+  }
+  // Lazy-require native modules so a broken install doesn't poison Sleuth startup
+  // for workspaces that never touch ask-self.
+  const Database = require('better-sqlite3');
+  const sqliteVec = require('sqlite-vec');
+  _db = new Database(DB_PATH, { readonly: true });
+  sqliteVec.load(_db);
+  return _db;
+}
+
+function getPrompts() {
+  if (_prompts) return _prompts;
+  _prompts = JSON.parse(fs.readFileSync(PROMPTS_PATH, 'utf8'));
+  return _prompts;
+}
+
+function assertTenancy(teamId) {
+  const allowed = process.env.NEOCHROME_TEAM_ID;
+  if (typeof allowed !== 'string' || allowed.length === 0) {
+    throw new TenancyError('NEOCHROME_TEAM_ID not configured');
+  }
+  if (typeof teamId !== 'string' || teamId.length === 0) {
+    throw new TenancyError('teamId argument required');
+  }
+  if (teamId !== allowed) {
+    throw new TenancyError('teamId does not match allowlist');
+  }
+}
+
+async function embedQuery(query) {
+  const apiKey = process.env.GOOGLE_API_KEY;
+  if (!apiKey) throw new Error('GOOGLE_API_KEY not set');
+  const endpoint = `https://generativelanguage.googleapis.com/v1beta/models/${EMBED_MODEL}:embedContent?key=${apiKey}`;
+  const res = await fetch(endpoint, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      model: `models/${EMBED_MODEL}`,
+      content: { parts: [{ text: query }] },
+      taskType: 'RETRIEVAL_QUERY',
+      outputDimensionality: EMBED_DIM,
+    }),
+  });
+  if (!res.ok) throw new Error(`Gemini embed ${res.status}: ${(await res.text()).slice(0, 300)}`);
+  const data = await res.json();
+  const values = data?.embedding?.values;
+  if (!Array.isArray(values) || values.length !== EMBED_DIM) {
+    throw new Error(`Gemini embed: unexpected shape, got ${values?.length} dims`);
+  }
+  return new Uint8Array(new Float32Array(values).buffer);
+}
+
+function knnSearch(db, queryVec, k = TOP_K) {
+  const hits = db.prepare(
+    'SELECT rowid, distance FROM chunks_vec WHERE embedding MATCH ? ORDER BY distance LIMIT ?'
+  ).all(queryVec, k);
+  if (hits.length === 0) return [];
+  const ids = hits.map((h) => Number(h.rowid));
+  const placeholders = ids.map(() => '?').join(',');
+  const rows = db.prepare(
+    `SELECT id, source, path, pr_number, version, priority, content FROM chunks WHERE id IN (${placeholders})`
+  ).all(...ids);
+  const byId = new Map(rows.map((r) => [Number(r.id), r]));
+  // Re-rank with priority boost: lower score is better.
+  // Drop hits whose metadata row is missing (e.g., partial/corrupt index) rather
+  // than spreading undefined into the result and throwing. Missing rows are logged
+  // once so an operator notices the drift instead of debugging silent gaps.
+  const dropped = [];
+  const ranked = [];
+  for (const h of hits) {
+    const row = byId.get(Number(h.rowid));
+    if (!row) {
+      dropped.push(h.rowid);
+      continue;
+    }
+    const score = h.distance - (row.priority ?? 1) * PRIORITY_BOOST;
+    ranked.push({ ...row, distance: h.distance, score });
+  }
+  if (dropped.length > 0) {
+    console.warn(`[rag] knnSearch: dropped ${dropped.length} hit(s) with missing metadata rows (rowids: ${dropped.join(', ')}). Rebuild the index with: npm run rag:ingest`);
+  }
+  return ranked.sort((a, b) => a.score - b.score);
+}
+
+async function synthesize(query, context, systemPrompt) {
+  const apiKey = process.env.GOOGLE_API_KEY;
+  const endpoint = `https://generativelanguage.googleapis.com/v1beta/models/${SYNTHESIS_MODEL}:generateContent?key=${apiKey}`;
+  const userMessage = `CONTEXT (retrieved from Sleuth's own corpus):\n\n${context}\n\n---\n\nQUESTION: ${query}`;
+  const body = {
+    system_instruction: { parts: [{ text: systemPrompt }] },
+    contents: [{ role: 'user', parts: [{ text: userMessage }] }],
+    generationConfig: { temperature: 0.3, maxOutputTokens: 1500 },
+  };
+  const res = await fetch(endpoint, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  });
+  if (!res.ok) throw new Error(`Gemini synthesis ${res.status}: ${(await res.text()).slice(0, 300)}`);
+  const data = await res.json();
+  const text = data?.candidates?.[0]?.content?.parts?.[0]?.text;
+  if (!text) throw new Error('Gemini synthesis: empty response');
+  return text;
+}
+
+/**
+ * Answer a question about Sleuth itself, grounded in the local RAG index.
+ * Strictly gated to the Neochrome workspace via NEOCHROME_TEAM_ID.
+ *
+ * @param {string} query - The question from the user.
+ * @param {string} teamId - The Slack team ID of the workspace the question came from.
+ * @returns {Promise<string>} - Formatted answer text to post back in Slack.
+ * @throws {TenancyError} - If teamId does not match NEOCHROME_TEAM_ID.
+ */
+async function askSelf(query, teamId) {
+  assertTenancy(teamId);
+  if (typeof query !== 'string' || query.trim().length === 0) {
+    throw new Error('query must be a non-empty string');
+  }
+  const prompts = getPrompts();
+  const db = getDb();
+  const queryVec = await embedQuery(query);
+  const hits = knnSearch(db, queryVec, TOP_K);
+  if (hits.length === 0) {
+    return "I couldn't find anything in my index for that question. Try `npm run rag:ingest` or rephrase.";
+  }
+  const context = formatContext(hits, MAX_CONTEXT_CHARS);
+  const answer = await synthesize(query, context, prompts.orchestrator_system);
+  const sourcesList = [...new Set(hits.slice(0, 8).map((h) =>
+    h.source === 'pr' ? `PR #${h.pr_number}` : h.path
+  ))];
+  return `${answer}\n\n_Sources consulted: ${sourcesList.join(', ')}_`;
+}
+
+module.exports = { askSelf, TenancyError };