Add MinHash fingerprinting and SIMILAR_TO edges for near-clone detection

Compute K=64 MinHash signatures from normalized AST node-type trigrams
during function extraction, then generate SIMILAR_TO edges via LSH
(b=32, r=2) for function pairs with Jaccard >= 0.95.

- src/simhash/minhash.{h,c}: MinHash compute, Jaccard, hex encode/decode,
  LSH index with band hashing for O(n) candidate generation
- src/pipeline/pass_similarity.c: post-pass reads fingerprints from node
  properties, builds LSH index, emits SIMILAR_TO edges with jaccard and
  same_file metadata. Same-language only, max 10 edges per node.
- internal/cbm/cbm.h: fingerprint fields on CBMDefinition
- internal/cbm/extract_defs.c: compute_fingerprint() hook at 3 extraction
  sites after complexity, skip functions with < 10 AST body nodes
- pass_definitions.c + pass_parallel.c: serialize fingerprint to "fp" hex
  in properties_json for both sequential and parallel pipeline paths
- pipeline.c + pipeline_incremental.c: register pass_similarity in both
  full and incremental post-pass lists
- tests/test_simhash.c: 28 tests across 4 suites (core, LSH, edge gen,
  pipeline integration with generated Go project + incremental)

Author: DeusData

Makefile.cbm

@@ -186,6 +186,10 @@ PIPELINE_SRCS = \
     src/pipeline/pass_compile_commands.c \
     src/pipeline/pass_infrascan.c \
     src/pipeline/pass_k8s.c \
+    src/pipeline/pass_similarity.c
+
+# SimHash / MinHash module
+SIMHASH_SRCS = src/simhash/minhash.c
 
 # Traces module (new)
 TRACES_SRCS = src/traces/traces.c
@@ -233,7 +237,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre
 YYJSON_SRC = vendored/yyjson/yyjson.c
 
 # All production sources
-PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
+PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
 EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
                   $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC)
 
@@ -301,7 +305,9 @@ TEST_SECURITY_SRCS = tests/test_security.c
 
 TEST_YAML_SRCS = tests/test_yaml.c
 
-ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_INTEGRATION_SRCS)
+TEST_SIMHASH_SRCS = tests/test_simhash.c
+
+ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_INTEGRATION_SRCS)
 
 
 # ── Build directories ────────────────────────────────────────────
@@ -505,8 +511,8 @@ SYSROOT_FLAG = $(if $(SYSROOT),-isysroot $(SYSROOT),)
 
 # Our source files (excluding vendored, grammars, tree-sitter runtime)
 LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) \
-            $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(TRACES_SRCS) \
-            $(WATCHER_SRCS) $(CLI_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) \
+            $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) \
+            $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) \
             $(SQLITE_WRITER_SRC) $(MAIN_SRC)
 LINT_HDRS = $(wildcard src/**/*.h src/*.h $(CBM_DIR)/*.h)
 LINT_TEST_SRCS = $(ALL_TEST_SRCS)

README.md

@@ -339,6 +339,19 @@ codebase-memory-mcp config set auto_index_limit 50000    # max files for auto-in
 codebase-memory-mcp config reset auto_index              # reset to default
 ```
 
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `CBM_CACHE_DIR` | `~/.cache/codebase-memory-mcp` | Override the database storage directory. All project indexes and config are stored here. |
+| `CBM_DIAGNOSTICS` | `false` | Set to `1` or `true` to enable periodic diagnostics output to `/tmp/cbm-diagnostics-<pid>.json`. |
+| `CBM_DOWNLOAD_URL` | *(GitHub releases)* | Override the download URL for updates. Used for testing or self-hosted deployments. |
+
+```bash
+# Store indexes in a custom directory
+export CBM_CACHE_DIR=~/my-projects/cbm-data
+```
+
 ## Custom File Extensions
 
 Map additional file extensions to supported languages via JSON config files. Useful for framework-specific extensions like `.blade.php` (Laravel) or `.mjs` (ES modules).

internal/cbm/cbm.h

@@ -103,6 +103,8 @@ typedef struct {
     const char *route_method;  // HTTP method from decorator (e.g., "POST") or NULL
     int complexity;            // cyclomatic complexity
     int lines;                 // body line count
+    uint32_t *fingerprint;     // MinHash fingerprint (arena-allocated, K values) or NULL
+    int fingerprint_k;         // number of hash values (CBM_MINHASH_K or 0)
     bool is_exported;
     bool is_abstract;
     bool is_test;

internal/cbm/extract_defs.c

@@ -3,6 +3,7 @@
 #include "helpers.h"
 #include "lang_specs.h"
 #include "foundation/constants.h"
+#include "simhash/minhash.h"
 #include "tree_sitter/api.h" // TSNode, ts_node_*
 #include <stdint.h>          // uint32_t
 #include <string.h>
@@ -28,6 +29,30 @@ enum {
     NESTED_CLASS_STACK_CAP = 128,
 };
 
+/* Compute MinHash fingerprint for a function body node and store in def.
+ * Sets def->fingerprint (arena-allocated) and def->fingerprint_k on success,
+ * leaves them NULL/0 if the body is too short. */
+static void compute_fingerprint(CBMExtractCtx *ctx, CBMDefinition *def, TSNode func_node) {
+    /* Find the function body child */
+    TSNode body = ts_node_child_by_field_name(func_node, TS_FIELD("body"));
+    if (ts_node_is_null(body)) {
+        /* Some languages use "block" or the function itself as the body */
+        body = func_node;
+    }
+    cbm_minhash_t result;
+    if (!cbm_minhash_compute(body, ctx->source, (int)ctx->language, &result)) {
+        return; /* Too short or empty — no fingerprint */
+    }
+    /* Arena-allocate the fingerprint array */
+    uint32_t *fp = cbm_arena_alloc(ctx->arena, CBM_MINHASH_K * sizeof(uint32_t));
+    if (!fp) {
+        return;
+    }
+    memcpy(fp, result.values, CBM_MINHASH_K * sizeof(uint32_t));
+    def->fingerprint = fp;
+    def->fingerprint_k = CBM_MINHASH_K;
+}
+
 // Tree-sitter row is 0-based; lines are 1-based.
 
 // Null-terminated array allocation: need count + 1 for terminator.
@@ -1421,6 +1446,9 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
         def.complexity = cbm_count_branching(node, spec->branching_node_types);
     }
 
+    // MinHash fingerprint
+    compute_fingerprint(ctx, &def, func_node);
+
     // JS/TS export detection
     if (ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT ||
         ctx->language == CBM_LANG_TSX) {
@@ -1855,6 +1883,9 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, const char *class_
         def.complexity = cbm_count_branching(child, spec->branching_node_types);
     }
 
+    // MinHash fingerprint
+    compute_fingerprint(ctx, &def, child);
+
     cbm_defs_push(&ctx->result->defs, a, def);
 }
 
@@ -1988,6 +2019,9 @@ static void extract_rust_impl(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
             def.complexity = cbm_count_branching(child, spec->branching_node_types);
         }
 
+        // MinHash fingerprint
+        compute_fingerprint(ctx, &def, child);
+
         cbm_defs_push(&ctx->result->defs, a, def);
     }
 }

src/pipeline/pass_definitions.c

@@ -20,6 +20,7 @@ enum { PD_RING = 4, PD_RING_MASK = 3, PD_JSON_MARGIN = 10, PD_ESC_MARGIN = 3, PD
 #include "foundation/log.h"
 #include "foundation/compat.h"
 #include "cbm.h"
+#include "simhash/minhash.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -191,6 +192,14 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
     append_json_string(buf, bufsize, &pos, "route_path", def->route_path);
     append_json_string(buf, bufsize, &pos, "route_method", def->route_method);
 
+    /* MinHash fingerprint — append if present and buffer has room. */
+    if (def->fingerprint && def->fingerprint_k > 0 &&
+        pos + CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD < bufsize) {
+        char fp_hex[CBM_MINHASH_HEX_BUF];
+        cbm_minhash_to_hex((const cbm_minhash_t *)def->fingerprint, fp_hex, sizeof(fp_hex));
+        append_json_string(buf, bufsize, &pos, "fp", fp_hex);
+    }
+
     if (pos < bufsize - SKIP_ONE) {
         buf[pos] = '}';
         buf[pos + SKIP_ONE] = '\0';

src/pipeline/pass_parallel.c

@@ -39,6 +39,7 @@ enum {
 #include "foundation/mem.h"
 #include "foundation/compat_regex.h"
 #include "cbm.h"
+#include "simhash/minhash.h"
 
 #include <stdatomic.h>
 #include <stdint.h>
@@ -212,6 +213,16 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
     append_json_str_array(buf, bufsize, &pos, "param_types", def->param_types);
     append_json_string(buf, bufsize, &pos, "route_path", def->route_path);
     append_json_string(buf, bufsize, &pos, "route_method", def->route_method);
+
+    /* MinHash fingerprint — append if present and buffer has room.
+     * Hex-encoded K=64 uint32 = 512 chars + key/quotes ≈ 520 chars. */
+    if (def->fingerprint && def->fingerprint_k > 0 &&
+        pos + CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD < bufsize) {
+        char fp_hex[CBM_MINHASH_HEX_BUF];
+        cbm_minhash_to_hex((const cbm_minhash_t *)def->fingerprint, fp_hex, sizeof(fp_hex));
+        append_json_string(buf, bufsize, &pos, "fp", fp_hex);
+    }
+
     if (pos < bufsize - SKIP_ONE) {
         buf[pos] = '}';
         buf[pos + SKIP_ONE] = '\0';