Merge pull request #29 from HiveForensics-AI/codex/implement-optional-local-semantic-reranking

Add optional Ollama-backed semantic sidecar reranking (lexical-first, additive)

Author: HiveForensicsAI

README.md

@@ -229,6 +229,27 @@ const hits = query(kb, 'react native bridge throttling', {
 });
 ```
 
+## Semantic sidecar workflow (Ollama, optional)
+
+Lexical retrieval is still the first-pass and default. Sidecars add optional local reranking over lexical top-N candidates (no vector DB, no `.knolo` format migration).
+
+```bash
+# 1) Build deterministic lexical pack
+knolo build
+
+# 2) Generate local semantic sidecar (requires Ollama running)
+knolo semantic:index --pack ./dist/knowledge.knolo --out ./dist/knowledge.knolo.semantic.json --model qwen3-embedding:4b
+
+# 3) Inspect and validate sidecar before query-time use
+knolo semantic:inspect --sidecar ./dist/knowledge.knolo.semantic.json
+knolo semantic:validate --pack ./dist/knowledge.knolo --sidecar ./dist/knowledge.knolo.semantic.json --model qwen3-embedding:4b
+```
+
+Troubleshooting:
+- If Ollama is not running, start it and ensure `http://localhost:11434` is reachable.
+- If model is missing, run `ollama pull qwen3-embedding:4b`.
+- If validate fails for fingerprint/model mismatch, regenerate sidecar with the current pack and exact model.
+
 ---
 
 # 🧠 Optional: Agent Metadata & Routing
@@ -443,4 +464,3 @@ const hits = query(pack, 'knolo determinism', {
 # 📄 License
 
 Apache-2.0 — see `LICENSE`
-

packages/cli/bin/knolo.mjs

@@ -27,7 +27,7 @@ const DEFAULT_CONFIG = {
 };
 const SUPPORTED_EXTENSIONS = new Set(['.md', '.txt', '.json']);
 const SKIP_DIRS = new Set(['node_modules', 'dist', '.git']);
-const SUBCOMMANDS = new Set(['init', 'add', 'build', 'query', 'dev']);
+const SUBCOMMANDS = new Set(['init', 'add', 'build', 'query', 'dev', 'semantic:index', 'semantic:inspect', 'semantic:validate']);
 
 function createError(message) {
   return new Error(message);
@@ -87,6 +87,9 @@ function printCommandHelp(command) {
     build: 'Usage: knolo build',
     query: 'Usage: knolo query <question> [--pack <path>] [--k <number>] [--json]',
     dev: 'Usage: knolo dev',
+    'semantic:index': 'Usage: knolo semantic:index --pack <path> [--out <path>] [--model <id>] [--endpoint <url>]',
+    'semantic:inspect': 'Usage: knolo semantic:inspect --sidecar <path>',
+    'semantic:validate': 'Usage: knolo semantic:validate --pack <path> --sidecar <path> --model <id>',
   };
   console.log(help[command] ?? 'Unknown command.');
 }
@@ -313,6 +316,79 @@ async function cmdQuery(core, args) {
   });
 }
 
+function parseKeyValueArgs(args) {
+  const out = {};
+  for (let i = 0; i < args.length; i++) {
+    const key = args[i];
+    if (!key.startsWith('--')) throw createError(`Unexpected argument: ${key}`);
+    out[key.slice(2)] = args[++i];
+  }
+  return out;
+}
+
+async function loadOllamaProvider() {
+  const mod = await tryImport(path.resolve(__dirname, '../../semantic-ollama/dist/index.js'));
+  if (mod?.OllamaEmbeddingProvider) return mod.OllamaEmbeddingProvider;
+  const pkg = await tryImport('@knolo/semantic-ollama');
+  if (pkg?.OllamaEmbeddingProvider) return pkg.OllamaEmbeddingProvider;
+  throw createError('Could not load @knolo/semantic-ollama. Build packages/semantic-ollama first.');
+}
+
+async function cmdSemanticIndex(core, args) {
+  const flags = parseKeyValueArgs(args);
+  const packPath = path.resolve(process.cwd(), flags.pack || 'dist/knowledge.knolo');
+  const outPath = path.resolve(process.cwd(), flags.out || `${packPath}.semantic.json`);
+  const modelId = flags.model || 'qwen3-embedding:4b';
+  const endpoint = flags.endpoint || 'http://localhost:11434';
+  if (!existsSync(packPath)) throw createError(`Pack file not found at ${path.relative(process.cwd(), packPath)}.`);
+
+  const bytes = Uint8Array.from(readFileSync(packPath));
+  const pack = await mountPackFromBytes(core, bytes);
+  const OllamaEmbeddingProvider = await loadOllamaProvider();
+  const provider = new OllamaEmbeddingProvider({ modelId, endpoint });
+  const vectors = await provider.embedTexts(pack.blocks);
+  const sidecar = {
+    version: 1,
+    packFingerprint: core.createPackFingerprint(pack),
+    modelId: provider.modelId,
+    dimension: vectors[0]?.length ?? 0,
+    metric: 'cosine',
+    createdAt: new Date().toISOString(),
+    blocks: vectors.map((vector, blockId) => ({ blockId, vector: Array.from(core.normalizeVector(vector)) })),
+  };
+  writeFileSync(outPath, core.serializeSidecar(sidecar));
+  console.log(`✔ wrote ${path.relative(process.cwd(), outPath)}`);
+}
+
+async function cmdSemanticInspect(core, args) {
+  const flags = parseKeyValueArgs(args);
+  const sidecarPath = path.resolve(process.cwd(), flags.sidecar);
+  const sidecar = core.parseSidecar(readFileSync(sidecarPath, 'utf8'));
+  console.log(JSON.stringify({
+    version: sidecar.version,
+    packFingerprint: sidecar.packFingerprint,
+    modelId: sidecar.modelId,
+    dimension: sidecar.dimension,
+    metric: sidecar.metric,
+    createdAt: sidecar.createdAt,
+    blocks: sidecar.blocks.length,
+  }, null, 2));
+}
+
+async function cmdSemanticValidate(core, args) {
+  const flags = parseKeyValueArgs(args);
+  const packPath = path.resolve(process.cwd(), flags.pack || 'dist/knowledge.knolo');
+  const sidecarPath = path.resolve(process.cwd(), flags.sidecar);
+  const modelId = flags.model;
+  if (!modelId) throw createError('semantic:validate requires --model <id>.');
+  const pack = await mountPackFromBytes(core, Uint8Array.from(readFileSync(packPath)));
+  const sidecar = core.parseSidecar(readFileSync(sidecarPath, 'utf8'));
+  core.validateSidecarForPack({ sidecar, pack, modelId });
+  if (sidecar.blocks.length !== pack.blocks.length) throw createError(`Semantic block count mismatch: sidecar=${sidecar.blocks.length}, pack=${pack.blocks.length}`);
+  if (sidecar.dimension <= 0) throw createError('Semantic sidecar dimension must be > 0.');
+  console.log('✔ semantic sidecar validation passed');
+}
+
 async function mountPackFromBytes(core, bytes) {
   try {
     return await core.mountPack({ bytes });
@@ -598,6 +674,9 @@ async function main() {
       if (command === 'build') return await cmdBuild(core);
       if (command === 'query') return await cmdQuery(core, commandArgs);
       if (command === 'dev') return await cmdDev(core);
+      if (command === 'semantic:index') return await cmdSemanticIndex(core, commandArgs);
+      if (command === 'semantic:inspect') return await cmdSemanticInspect(core, commandArgs);
+      if (command === 'semantic:validate') return await cmdSemanticValidate(core, commandArgs);
     }
 
     if (command.startsWith('-')) throw createError(`Unknown option: ${command}`);

packages/cli/test/cli.test.mjs

@@ -4,6 +4,7 @@ import { mkdtempSync, existsSync, mkdirSync, writeFileSync, readFileSync } from
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { execFileSync } from 'node:child_process';
+import { pathToFileURL } from 'node:url';
 
 const cliPath = path.resolve(process.cwd(), 'bin/knolo.mjs');
 const cliPackageJson = JSON.parse(
@@ -91,3 +92,33 @@ test('add updates existing source path', () => {
   const config = JSON.parse(readFileSync(path.join(cwd, 'knolo.config.json'), 'utf8'));
   assert.equal(config.sources[0].path, './knowledge-base');
 });
+
+test('semantic:validate succeeds for matching pack/model and fails on mismatch', async () => {
+  const cwd = mkdtempSync(path.join(tmpdir(), 'knolo-cli-sem-validate-'));
+  runCli(['init'], cwd);
+  runCli(['build'], cwd);
+
+  const coreModule = await import(pathToFileURL(path.resolve(process.cwd(), '../core/dist/index.js')).href);
+  const packPath = path.join(cwd, 'dist/knowledge.knolo');
+  const packBytes = readFileSync(packPath);
+  const pack = await coreModule.mountPack({ src: Uint8Array.from(packBytes) });
+  const sidecarPath = path.join(cwd, 'dist/knowledge.knolo.semantic.json');
+  const sidecar = {
+    version: 1,
+    packFingerprint: coreModule.createPackFingerprint(pack),
+    modelId: 'qwen3-embedding:4b',
+    dimension: 3,
+    metric: 'cosine',
+    createdAt: new Date().toISOString(),
+    blocks: pack.blocks.map((_, blockId) => ({ blockId, vector: [1, 0, 0] })),
+  };
+  writeFileSync(sidecarPath, coreModule.serializeSidecar(sidecar), 'utf8');
+
+  const output = runCli(['semantic:validate', '--pack', './dist/knowledge.knolo', '--sidecar', './dist/knowledge.knolo.semantic.json', '--model', 'qwen3-embedding:4b'], cwd);
+  assert.match(output, /validation passed/);
+
+  assert.throws(
+    () => runCli(['semantic:validate', '--pack', './dist/knowledge.knolo', '--sidecar', './dist/knowledge.knolo.semantic.json', '--model', 'other-model'], cwd),
+    /Semantic model mismatch/
+  );
+});

packages/core/scripts/test.mjs

@@ -37,6 +37,12 @@ import {
   mergeClaimGraphLogs,
   applyClaimGraphLog,
   expandQueryWithGraph,
+  cosineSimilarity,
+  normalizeVector,
+  createPackFingerprint,
+  serializeSidecar,
+  parseSidecar,
+  validateSidecarForPack,
 } from '../dist/index.js';
 import { mountPack as mountPackNode } from '../dist/node.js';
 
@@ -613,6 +619,132 @@ async function testSemanticRerankErrorAndDefaults() {
   );
 }
 
+async function testSemanticSidecarRerankAndValidation() {
+  const docs = [
+    { id: 'a', text: 'alpha beta alpha beta alpha beta river stone' },
+    { id: 'b', text: 'alpha beta solar wind' },
+  ];
+  const pack = await mountPack({ src: await buildPack(docs) });
+  const sidecar = {
+    version: 1,
+    packFingerprint: createPackFingerprint(pack),
+    modelId: 'qwen3-embedding:4b',
+    dimension: 2,
+    metric: 'cosine',
+    createdAt: new Date().toISOString(),
+    blocks: [
+      { blockId: 0, vector: [1, 0] },
+      { blockId: 1, vector: [0, 1] },
+    ],
+  };
+
+  const lexical = query(pack, 'alpha beta', { topK: 2, queryExpansion: { enabled: false } });
+  const reranked = query(pack, 'alpha beta', {
+    topK: 2,
+    queryExpansion: { enabled: false },
+    semantic: {
+      enabled: true,
+      sidecarPath: serializeSidecar(sidecar),
+      provider: { type: 'ollama', modelId: 'qwen3-embedding:4b' },
+      queryEmbedding: new Float32Array([0, 1]),
+      force: true,
+      blend: { enabled: false },
+    },
+  });
+
+  assert.notEqual(reranked[0]?.source, lexical[0]?.source, 'expected sidecar rerank to update ordering');
+  assert.equal(reranked[0]?.evidence?.retrieval, 'hybrid');
+
+  assert.throws(
+    () => validateSidecarForPack({ sidecar: { ...sidecar, modelId: 'other' }, pack, modelId: 'qwen3-embedding:4b' }),
+    /Semantic model mismatch/
+  );
+  assert.throws(
+    () => validateSidecarForPack({ sidecar: { ...sidecar, packFingerprint: 'fnv1a-deadbeef' }, pack, modelId: 'qwen3-embedding:4b' }),
+    /pack fingerprint mismatch/
+  );
+
+  const loaded = parseSidecar(serializeSidecar(sidecar));
+  assert.deepEqual(loaded, sidecar, 'expected semantic sidecar round trip to remain stable');
+}
+
+async function testSemanticEvidenceScoresRemainCorrectAfterRerank() {
+  const docs = [
+    { id: 'lex-a', text: 'alpha beta alpha beta alpha beta river stone' },
+    { id: 'lex-b', text: 'alpha beta solar wind' },
+  ];
+  const pack = await mountPack({
+    src: await buildPack(docs, {
+      semantic: {
+        enabled: true,
+        modelId: 'test-model',
+        embeddings: [new Float32Array([1, 0]), new Float32Array([0, 1])],
+        quantization: { type: 'int8_l2norm', perVectorScale: true },
+      },
+    }),
+  });
+
+  const lexical = query(pack, 'alpha beta', {
+    topK: 2,
+    queryExpansion: { enabled: false },
+  });
+  const lexicalScores = new Map(lexical.map((h) => [h.blockId, h.evidence?.lexicalScore ?? h.score]));
+  const reranked = query(pack, 'alpha beta', {
+    topK: 2,
+    queryExpansion: { enabled: false },
+    semantic: {
+      enabled: true,
+      queryEmbedding: new Float32Array([0, 1]),
+      force: true,
+      blend: { enabled: true, wLex: 0.5, wSem: 0.5 },
+    },
+  });
+
+  assert.notEqual(
+    reranked[0]?.source,
+    lexical[0]?.source,
+    'expected semantic rerank to change ordering'
+  );
+  for (const hit of reranked) {
+    const before = lexicalScores.get(hit.blockId);
+    assert.equal(
+      hit.evidence?.lexicalScore,
+      before,
+      'expected evidence.lexicalScore to preserve pre-rerank lexical score'
+    );
+    assert.equal(hit.evidence?.retrieval, 'hybrid');
+    assert.equal(typeof hit.evidence?.semanticScore, 'number');
+    assert.equal(typeof hit.evidence?.blendedScore, 'number');
+  }
+}
+
+async function testLexicalOnlyEvidenceRemainsUnchanged() {
+  const docs = [
+    { id: 'a', text: 'alpha beta gamma' },
+    { id: 'b', text: 'alpha beta delta' },
+  ];
+  const pack = await mountPack({ src: await buildPack(docs) });
+  const hits = query(pack, 'alpha beta', {
+    topK: 2,
+    queryExpansion: { enabled: false },
+  });
+  assert.ok(hits.length > 0, 'expected lexical query to return hits');
+  for (const hit of hits) {
+    assert.equal(hit.evidence?.retrieval, 'lexical');
+    assert.equal(typeof hit.evidence?.lexicalScore, 'number');
+    assert.equal(hit.evidence?.semanticScore, undefined);
+    assert.equal(hit.evidence?.blendedScore, undefined);
+  }
+}
+
+async function testCosineHelpers() {
+  const a = normalizeVector(new Float32Array([3, 4]));
+  const b = normalizeVector(new Float32Array([3, 4]));
+  const c = normalizeVector(new Float32Array([4, -3]));
+  assert.ok(Math.abs(cosineSimilarity(a, b) - 1) < 1e-6, 'expected same vector cosine to be 1');
+  assert.ok(Math.abs(cosineSimilarity(a, c)) < 1e-6, 'expected orthogonal vector cosine to be ~0');
+}
+
 async function testSemanticFixtureAndHelpers() {
   const pack = await buildSemanticFixturePack();
   assert.ok(
@@ -1638,6 +1770,10 @@ await testLexConfidenceDeterministic();
 await testSemanticRerankLowConfidence();
 await testSemanticRerankRespectsConfidenceAndForce();
 await testSemanticRerankErrorAndDefaults();
+await testSemanticSidecarRerankAndValidation();
+await testSemanticEvidenceScoresRemainCorrectAfterRerank();
+await testLexicalOnlyEvidenceRemainsUnchanged();
+await testCosineHelpers();
 await testSmartQuotePhrase();
 await testFirstBlockRetrieval();
 await testNearDuplicateDedupe();

packages/core/src/index.ts

@@ -13,6 +13,15 @@ export {
   encodeScaleF16,
   decodeScaleF16,
 } from './semantic.js';
+export { cosineSimilarity, normalizeVector } from './semantic/cosine.js';
+export {
+  createPackFingerprint,
+  serializeSidecar,
+  parseSidecar,
+  validateSidecarForPack,
+} from './semantic/sidecar.js';
+export { rerankCandidates } from './semantic/rerank.js';
+export { assertProviderCompatible, ensureProviderModelId } from './semantic/provider.js';
 export {
   listAgents,
   getAgent,
@@ -39,6 +48,7 @@ export {
 export { expandQueryWithGraph } from './graph/query_expand.js';
 export type { MountOptions, PackMeta, Pack } from './pack.runtime.js';
 export type { QueryOptions, Hit } from './query.js';
+export type { EmbeddingProvider, SemanticSidecar, SemanticQueryOptions, RetrievalEvidence } from './semantic/types.js';
 export type { ContextPatch } from './patch.js';
 export type { BuildInputDoc, BuildPackOptions } from './builder.js';
 export type {