From a509ee7616b7bd621d76415c31c78d45c43e8590 Mon Sep 17 00:00:00 2001 From: Daniel Adewale Date: Sat, 18 Jul 2026 14:15:58 +0100 Subject: [PATCH] feat: enhance code search and visualization tools - Added `search_code` tool for full-text search over indexed files, replacing traditional grep. - Introduced `trace_calls` tool to analyze call graphs, providing insights into function interactions. - Updated `find_symbol` tool to support natural-language queries alongside exact names and regex patterns. - Enhanced Atlas visualization to include new node types (`method`) and link kinds (`extends`, `implements`). - Improved rendering logic for nodes and links in the Atlas view, adjusting sizes and colors for better clarity. - Updated API types to include new symbol kinds and link types, ensuring consistency across the application. --- README.md | 6 +- bench/estimate.ts | 165 ++++++++++++++ core/src/bm25.ts | 123 +++++++++++ core/src/build.ts | 109 +++++++++- core/src/cancellation.ts | 74 +++++++ core/src/index.ts | 12 ++ core/src/lock.ts | 72 +++++++ core/src/nodeid.ts | 153 +++++++++++++ core/src/parse/grammars/index.ts | 7 + core/src/parse/grammars/java.ts | 10 +- core/src/parse/grammars/javascript.ts | 35 ++- core/src/parse/grammars/python.ts | 23 +- core/src/parse/grammars/typescript.ts | 46 +++- core/src/parse/index.ts | 30 ++- core/src/parse/treesitter.ts | 26 ++- core/src/position.ts | 62 ++++++ core/src/revision.ts | 84 ++++++++ core/src/roots.ts | 84 ++++++++ core/src/search.ts | 251 ++++++++++++++++++++++ core/src/trace.ts | 210 ++++++++++++++++++ core/src/types.ts | 92 +++++++- core/tests/concurrency.test.ts | 151 +++++++++++++ core/tests/fixtures/repo/sample.ts | 18 ++ core/tests/fixtures/utf16-sample.ts | 29 +++ core/tests/multiroot.test.ts | 95 ++++++++ core/tests/nodeid.test.ts | 98 +++++++++ core/tests/position.test.ts | 77 +++++++ core/tests/ranges.test.ts | 75 +++++++ core/tests/revision.test.ts | 86 ++++++++ docs/GRAPH.md | 8 +- mcp/README.md | 6 +- mcp/hooks/openvisio-gate.mjs | 39 ++-- mcp/smoke.mjs | 14 +- mcp/src/adapter.ts | Bin 7916 -> 8221 bytes mcp/src/answerer.ts | 3 + mcp/src/server.ts | 15 +- mcp/src/tools.ts | 92 +++++++- ui/components/graph/AtlasView.tsx | 18 +- ui/lib/api/types.ts | 2 +- ui/lib/graph/atlas.ts | 169 +++++++++------ viewer/src/components/graph/AtlasView.tsx | 18 +- viewer/src/lib/api/types.ts | 2 +- viewer/src/lib/graph/atlas.ts | 169 +++++++++------ 43 files changed, 2641 insertions(+), 217 deletions(-) create mode 100644 core/src/bm25.ts create mode 100644 core/src/cancellation.ts create mode 100644 core/src/lock.ts create mode 100644 core/src/nodeid.ts create mode 100644 core/src/position.ts create mode 100644 core/src/revision.ts create mode 100644 core/src/roots.ts create mode 100644 core/src/search.ts create mode 100644 core/src/trace.ts create mode 100644 core/tests/concurrency.test.ts create mode 100644 core/tests/fixtures/repo/sample.ts create mode 100644 core/tests/fixtures/utf16-sample.ts create mode 100644 core/tests/multiroot.test.ts create mode 100644 core/tests/nodeid.test.ts create mode 100644 core/tests/position.test.ts create mode 100644 core/tests/ranges.test.ts create mode 100644 core/tests/revision.test.ts diff --git a/README.md b/README.md index dd0ee83..99d6424 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,11 @@ file-crawling an agent does just to find where the relevant code lives. |------|--------------| | `resolve_context` | task-ranked skeleton + neighborhoods of the most relevant files (call first) | | `get_repo_skeleton` | the whole ranked repo map | -| `find_symbol` | locate a function/class/type → signature + `path:line` | +| `find_symbol` | locate a function/class/type by name, regex, or **natural-language query** (BM25) → signature + `path:line` | +| `search_code` | full-text search (the grep replacement) → ranked, symbol-annotated, anchored hits | +| `trace_calls` | walk the call graph — who calls a function / what it calls (the "grep for callers" replacement) | | `get_neighborhood` | local import subgraph around a file/symbol | -| `get_dependents` | who imports this (impact analysis) | +| `get_dependents` | who imports this (file-level impact analysis) | | `get_hotspots` | churn × centrality refactor/risk candidates | Every line carries a `path:line` anchor, so agents read only the slice they need. diff --git a/bench/estimate.ts b/bench/estimate.ts index ff73af3..938cab9 100644 --- a/bench/estimate.ts +++ b/bench/estimate.ts @@ -18,10 +18,17 @@ import * as path from 'node:path' import { buildGraph, buildSkeleton, + computeCentrality, estimateTokens, + rankSymbols, resolveContext, scanRepo, + searchContent, + tokenize, + traceCalls, + type Centrality, type CodeGraph, + type CodeSymbol, type ScannedFile, } from '@openvisio/core' @@ -99,6 +106,138 @@ function fmt(n: number): string { return n >= 1000 ? `${(n / 1000).toFixed(1)}K` : `${n}` } +// --------------------------------------------------------------------------- +// Targeted-operation baselines. resolve_context models "start a task"; these +// model the three grep-shaped LOOKUPS the new tools replace — searching for a +// string, tracing callers, and finding a symbol by concept. For each we count +// the tokens a grep agent burns (read whole every file that matches the term) +// vs the exact tokens the tool returns for the same question. +// --------------------------------------------------------------------------- + +/** Files whose content contains `term` (case-insensitive), ranked by hit count + * and capped like a real agent's read budget — the grep-then-read baseline. */ +function filesContainingTerm( + term: string, + graph: CodeGraph, + contentByPath: Map, +): number[] { + const needle = term.toLowerCase() + const scored: { id: number; hits: number }[] = [] + for (const f of graph.files) { + const content = contentByPath.get(f.path) + if (content === undefined) continue + const hay = content.toLowerCase() + if (!hay.includes(needle)) continue + // Count occurrences cheaply (split is fine for an estimate). + scored.push({ id: f.id, hits: hay.split(needle).length - 1 }) + } + scored.sort((a, b) => b.hits - a.hits || a.id - b.id) + return scored.slice(0, MAX_GREP_FILES).map((s) => s.id) +} + +/** Symbol ids that are the TARGET of at least one call edge — i.e. have callers + * worth tracing (so the trace-baseline isn't measured on dead leaves). */ +function symbolsWithCallers(graph: CodeGraph): Set { + const out = new Set() + for (const e of graph.edges) if (e.kind === 'calls') out.add(e.targetId) + return out +} + +/** Deterministically pick probe symbols: most import-central first, exported and + * specifically-named preferred, unique names. These stand in for the symbols an + * agent actually looks up. */ +function pickProbes(graph: CodeGraph, centrality: Centrality, count: number): CodeSymbol[] { + const seen = new Set() + return [...graph.symbols] + .filter((s) => s.name.length >= 5 && /^[A-Za-z_]/.test(s.name)) + .sort((a, b) => { + const sa = centrality.scoreByFile.get(a.fileId) ?? 0 + const sb = centrality.scoreByFile.get(b.fileId) ?? 0 + if (sb !== sa) return sb - sa + if (a.exported !== b.exported) return a.exported ? -1 : 1 + const pa = graph.filesById.get(a.fileId)?.path ?? '' + const pb = graph.filesById.get(b.fileId)?.path ?? '' + if (pa !== pb) return pa.localeCompare(pb) + return a.startLine - b.startLine + }) + .filter((s) => (seen.has(s.name) ? false : (seen.add(s.name), true))) + .slice(0, count) +} + +interface OpStat { + ov: number + base: number + n: number +} + +/** Run the three grep-replacement operations over the probe set and total the + * tokens each side spends. Everything is deterministic. */ +function measureOperations( + graph: CodeGraph, + centrality: Centrality, + contentByPath: Map, +): { search: OpStat; trace: OpStat; find: OpStat } { + const probes = pickProbes(graph, centrality, 12) + const callable = symbolsWithCallers(graph) + const search: OpStat = { ov: 0, base: 0, n: 0 } + const trace: OpStat = { ov: 0, base: 0, n: 0 } + const find: OpStat = { ov: 0, base: 0, n: 0 } + + for (const sym of probes) { + // 1) search_code: grep a string → read matching files whole vs matched lines. + const searchBase = tokensOfFiles(filesContainingTerm(sym.name, graph, contentByPath), graph, contentByPath) + if (searchBase > 0) { + search.ov += estimateTokens(searchContent(graph, { query: sym.name, centrality }).text) + search.base += searchBase + search.n++ + } + + // 2) trace_calls: "who calls X" → grep the name + read every hit vs the tree. + if (callable.has(sym.id)) { + const traceBase = tokensOfFiles(filesContainingTerm(sym.name, graph, contentByPath), graph, contentByPath) + if (traceBase > 0) { + trace.ov += estimateTokens( + traceCalls(graph, { symbolName: sym.name, direction: 'callers', centrality }).text, + ) + trace.base += traceBase + trace.n++ + } + } + + // 3) find_symbol (BM25): concept query (the tokenized name) → grep each term + // and read files matching ≥2 terms, vs the ranked signature list. + const terms = [...new Set(tokenize(sym.name))] + if (terms.length >= 2) { + const matchIds = new Set() + for (const f of graph.files) { + const content = contentByPath.get(f.path) + if (content === undefined) continue + const hay = (f.path + '\n' + content).toLowerCase() + let hit = 0 + for (const t of terms) if (hay.includes(t)) hit++ + if (hit >= 2) matchIds.add(f.id) + } + const findBase = tokensOfFiles([...matchIds].slice(0, MAX_GREP_FILES), graph, contentByPath) + if (findBase > 0) { + const hits = rankSymbols(graph, terms.join(' '), { centrality, limit: 8 }) + // find_symbol returns signature + anchor per hit (the discovery surface). + const rendered = hits + .map((h) => `${h.symbol.signature} @${h.file.path}:${h.symbol.startLine}`) + .join('\n') + find.ov += Math.min(estimateTokens(rendered), 800) // tool caps at ~800 tokens + find.base += findBase + find.n++ + } + } + } + return { search, trace, find } +} + +/** Percent token reduction for an operation stat (0 when no samples). */ +function reduction(s: OpStat): number { + return s.base > 0 ? (1 - s.ov / s.base) * 100 : 0 +} + async function main(): Promise { const argv = process.argv.slice(2) const positional = argv.filter((a) => !a.startsWith('--')) @@ -124,11 +263,19 @@ async function main(): Promise { if (graph.fileIdByPath.has(sf.relPath)) contentByPath.set(sf.relPath, sf.content) } + const centrality = computeCentrality(graph) + // Whole-repo "understand this codebase" priming comparison. const repoTokens = tokensOfFiles(graph.files.map((f) => f.id), graph, contentByPath) const skeleton = buildSkeleton(graph, { budgetTokens: 1500 }) const skeletonTokens = estimateTokens(skeleton.text) + // Targeted grep-replacement operations (search_code / trace_calls / find_symbol). + const ops = measureOperations(graph, centrality, contentByPath) + const opsBase = ops.search.base + ops.trace.base + ops.find.base + const opsOv = ops.search.ov + ops.trace.ov + ops.find.ov + const opsReduction = opsBase > 0 ? (1 - opsOv / opsBase) * 100 : 0 + const rows: string[] = [] let sumOv = 0 let sumBase = 0 @@ -174,6 +321,20 @@ async function main(): Promise { ...rows, `| **TOTAL** | **${fmt(sumOv)}** | **${fmt(sumBase)}** | **${totalRatio.toFixed(1)}×** | |`, '', + '## Targeted lookups (the grep-replacement tools)', + '', + `> Models the three grep-shaped operations the tools replace, over ${ops.search.n}`, + `> deterministically-sampled probe symbols (most import-central first). Baseline =`, + `> tokens of every file a grep agent reads whole to answer the same question;`, + `> OpenVisio = the exact tokens the tool returns.`, + '', + '| operation | replaces | OpenVisio | grep baseline | reduction | samples |', + '|---|---|---|---|---|---|', + `| \`search_code\` | grep a string, read hit files | ${fmt(ops.search.ov)} | ${fmt(ops.search.base)} | ${reduction(ops.search).toFixed(0)}% | ${ops.search.n} |`, + `| \`trace_calls\` | grep "who calls X", read hits | ${fmt(ops.trace.ov)} | ${fmt(ops.trace.base)} | ${reduction(ops.trace).toFixed(0)}% | ${ops.trace.n} |`, + `| \`find_symbol\` (BM25) | grep a concept, read hits | ${fmt(ops.find.ov)} | ${fmt(ops.find.base)} | ${reduction(ops.find).toFixed(0)}% | ${ops.find.n} |`, + `| **COMBINED** | | **${fmt(opsOv)}** | **${fmt(opsBase)}** | **${opsReduction.toFixed(0)}%** | |`, + '', '## Method & honesty notes', '', `- Baseline = Σ tokens of files a no-graph agent opens: keyword-grep hits`, @@ -183,6 +344,10 @@ async function main(): Promise { ` re-process context every agent-loop turn (Codex ~3–5×), none of which is`, ` counted here. It also ignores the per-turn tool-definition tax.`, `- OpenVisio cost = exact size of \`resolve_context\` output (one call).`, + `- **Targeted lookups**: probes are the ${12} most-central uniquely-named symbols. The`, + ` grep baseline reads whole every file whose text contains the term (capped at`, + ` ${MAX_GREP_FILES}); the tool cost is its actual returned text. Conservative — it counts`, + ` one grep+read pass, not the re-greps a real agent runs on ambiguous names.`, `- It does **not** model answer quality. Graph-first trades a few quality points`, ` for the token saving (see the research doc); the agent can always fall back`, ` to a real file read via the anchors OpenVisio returns.`, diff --git a/core/src/bm25.ts b/core/src/bm25.ts new file mode 100644 index 0000000..9011f7c --- /dev/null +++ b/core/src/bm25.ts @@ -0,0 +1,123 @@ +// BM25 symbol ranking — natural-language code discovery without a vector index. +// find_symbol's exact/regex modes only help when you already know the +// identifier. The expensive failure is the guess→miss→grep-again loop when you +// DON'T ("the function that updates the cloud client" → `updateCloudClient`). +// BM25 closes it: tokenize the query and the symbol corpus (name + signature + +// path) with camelCase/snake_case splitting, score by term frequency × rarity, +// and rank — so multi-word, order-independent, sub-identifier queries land on +// the right symbol in one call. Deterministic, dependency-free, no embeddings. + +import { computeCentrality, type Centrality } from './rank.js' +import type { CodeFile, CodeGraph, CodeSymbol } from './types.js' + +/** Split an identifier/phrase into lowercase word tokens: whitespace, punctuation, + * camelCase, and snake_case all break. `updateCloudClient` → update, cloud, client. */ +export function tokenize(text: string): string[] { + const out: string[] = [] + for (const raw of text.split(/[^A-Za-z0-9]+/)) { + if (!raw) continue + // Split camelCase / PascalCase / digit boundaries. + for (const piece of raw.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/([A-Za-z])([0-9])/g, '$1 $2').split(/\s+/)) { + const t = piece.toLowerCase() + if (t.length >= 2) out.push(t) + } + } + return out +} + +/** Structural priors mirroring search_graph: callable code ranks above types, + * types above bare consts. Applied as a small multiplicative boost. */ +const KIND_BOOST: Record = { + function: 1.25, + method: 1.2, + class: 1.12, + interface: 1.08, + type: 1.05, + const: 1.0, +} + +export interface RankedSymbolHit { + symbol: CodeSymbol + file: CodeFile + score: number +} + +export interface Bm25Options { + limit?: number + centrality?: Centrality + /** BM25 term-frequency saturation (default 1.5). */ + k1?: number + /** BM25 length normalization (default 0.75). */ + b?: number +} + +/** + * Rank symbols against a natural-language query with BM25 over their + * name+signature+path tokens, then apply light structural (kind) and centrality + * boosts so the load-bearing, callable hit wins ties. Returns [] when the query + * has no usable terms. + */ +export function rankSymbols(graph: CodeGraph, query: string, opts: Bm25Options = {}): RankedSymbolHit[] { + const qTerms = [...new Set(tokenize(query))] + if (qTerms.length === 0) return [] + const centrality = opts.centrality ?? computeCentrality(graph) + const k1 = opts.k1 ?? 1.5 + const b = opts.b ?? 0.75 + const limit = opts.limit ?? 25 + + // Build the per-symbol token documents once. + type Doc = { sym: CodeSymbol; file: CodeFile; tf: Map; len: number } + const docs: Doc[] = [] + const df = new Map() + let totalLen = 0 + for (const sym of graph.symbols) { + const file = graph.filesById.get(sym.fileId) + if (!file) continue + const tokens = tokenize(`${sym.name} ${sym.signature} ${file.path}`) + if (tokens.length === 0) continue + const tf = new Map() + for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1) + // Document frequency counts each query term at most once per doc. + for (const t of qTerms) if (tf.has(t)) df.set(t, (df.get(t) ?? 0) + 1) + docs.push({ sym, file, tf, len: tokens.length }) + totalLen += tokens.length + } + if (docs.length === 0) return [] + const avgLen = totalLen / docs.length + const N = docs.length + + // Precompute idf per query term (BM25 idf, floored at 0 so ubiquitous terms + // don't push scores negative). + const idf = new Map() + for (const t of qTerms) { + const n = df.get(t) ?? 0 + idf.set(t, Math.max(0, Math.log(1 + (N - n + 0.5) / (n + 0.5)))) + } + + const hits: RankedSymbolHit[] = [] + for (const d of docs) { + let score = 0 + let matched = 0 + for (const t of qTerms) { + const f = d.tf.get(t) + if (!f) continue + matched++ + const w = idf.get(t) ?? 0 + score += w * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * d.len) / avgLen))) + } + if (matched === 0) continue + // Reward covering more distinct query terms; apply kind + centrality boosts. + const coverage = matched / qTerms.length + const cen = 1 + (centrality.scoreByFile.get(d.file.id) ?? 0) + score *= KIND_BOOST[d.sym.kind] * (0.5 + 0.5 * coverage) * cen + hits.push({ symbol: d.sym, file: d.file, score }) + } + + hits.sort((a, b2) => { + if (b2.score !== a.score) return b2.score - a.score + if (a.symbol.exported !== b2.symbol.exported) return a.symbol.exported ? -1 : 1 + if (a.file.path !== b2.file.path) return a.file.path.localeCompare(b2.file.path) + return a.symbol.startLine - b2.symbol.startLine + }) + return hits.slice(0, limit) +} diff --git a/core/src/build.ts b/core/src/build.ts index 93f0d7d..ce9f17a 100644 --- a/core/src/build.ts +++ b/core/src/build.ts @@ -12,6 +12,9 @@ import { scanRepo, type ScanOptions, type ScannedFile } from './scan.js' import { parseJsonc } from './jsonc.js' import { grammarForFile, loadGrammars } from './parse/treesitter.js' import { GRAMMARS, type TsAliases } from './parse/grammars/index.js' +import { computeLineStarts } from './position.js' +import { assignNodeIds, rootIdFor } from './nodeid.js' +import { assignRevisions, newRevisionState, type RevisionState } from './revision.js' import type { Store } from './store.js' import type { CodeEdge, @@ -128,6 +131,9 @@ export interface AssembleContext { idByPath?: Map aliases?: TsAliases parseTimeoutMs?: number + /** Baseline revision stamped on every node (default 1). The Indexer overrides + * per-node via assignRevisions; standalone builds stay uniform at this value. */ + revision?: number } export async function assembleGraph( @@ -136,6 +142,7 @@ export async function assembleGraph( ctx: AssembleContext = {}, ): Promise { const cache = ctx.cache + const buildRevision = ctx.revision ?? 1 const idByPath = ctx.idByPath ?? new Map() let nextFileId = 0 for (const id of idByPath.values()) nextFileId = Math.max(nextFileId, id + 1) @@ -155,6 +162,7 @@ export async function assembleGraph( const filesById = new Map() const rawImportsByFile = new Map() const rawCallsByFile = new Map() + const rawInheritsByFile = new Map() let nextSymbolId = 0 console.error(`[build] phase 1: assigning IDs to ${scanned.length} files`) @@ -162,11 +170,16 @@ export async function assembleGraph( const fileId = allocId(sf.relPath) files.push({ id: fileId, + nodeId: '', // assigned by assignNodeIds once all nodes exist path: sf.relPath, language: sf.language, loc: sf.loc, sha: sf.sha, lastModified: sf.lastModified, + // UTF-16 line index for offset->Position conversion (symbol ranges). Cheap + // to recompute; held in memory only, so it never touches the parse cache. + lineStarts: computeLineStarts(sf.content), + revision: buildRevision, // per-node revision refined by assignRevisions }) filesById.set(fileId, files[files.length - 1]!) fileIdByPath.set(sf.relPath, fileId) @@ -202,7 +215,7 @@ export async function assembleGraph( parsed = await parseFile(sf.relPath, sf.content, ctx.parseTimeoutMs) } catch (err) { console.error(`[build] parse error for ${sf.relPath}: ${err}`) - parsed = { symbols: [], imports: [], calls: [] } + parsed = { symbols: [], imports: [], calls: [], inherits: [] } } if (cache && 'get' in cache && 'set' in cache) { const store = cache as Store @@ -230,13 +243,14 @@ export async function assembleGraph( const fileId = fileIdByPath.get(sf.relPath)! const fileSymbols: CodeSymbol[] = [] for (const s of parsed.symbols) { - const sym: CodeSymbol = { id: nextSymbolId++, fileId, ...s } + const sym: CodeSymbol = { id: nextSymbolId++, nodeId: '', revision: buildRevision, fileId, ...s } symbols.push(sym) fileSymbols.push(sym) } symbolsByFile.set(fileId, fileSymbols) rawImportsByFile.set(fileId, parsed.imports) rawCallsByFile.set(fileId, parsed.calls ?? []) + rawInheritsByFile.set(fileId, parsed.inherits ?? []) if (grammarForFile(sf.relPath) && parsed.symbols.length === 0 && parsed.imports.length === 0) { emptyCount++ } @@ -245,6 +259,31 @@ export async function assembleGraph( } console.error(`[build] phase 2 done: ${symbols.length} symbols extracted`) + // Language-agnostic method promotion: a `function` whose smallest strict + // container (by UTF-16 fullRange) is a `class` is really a method. This gives + // Python/Ruby/etc. the Function/Method split for free — grammars that already + // emit `method` (TS/JS/Java) are untouched (they aren't `function`), and a + // function nested in another function stays a function. + let promoted = 0 + for (const fileSymbols of symbolsByFile.values()) { + for (const s of fileSymbols) { + if (s.kind !== 'function') continue + let best: CodeSymbol | null = null + for (const c of fileSymbols) { + if (c === s) continue + const enclosesInclusive = c.fullRange[0] <= s.fullRange[0] && s.fullRange[1] <= c.fullRange[1] + const strictlyLarger = c.fullRange[0] < s.fullRange[0] || s.fullRange[1] < c.fullRange[1] + if (!enclosesInclusive || !strictlyLarger) continue + if (!best || c.fullRange[1] - c.fullRange[0] < best.fullRange[1] - best.fullRange[0]) best = c + } + if (best && best.kind === 'class') { + s.kind = 'method' + promoted++ + } + } + } + if (promoted > 0) console.error(`[build] promoted ${promoted} nested function(s) to methods`) + console.error(`[build] resolving imports across ${files.length} files...`) const bySet = new Set(fileIdByPath.keys()) @@ -300,6 +339,8 @@ export async function assembleGraph( }) const edges: CodeEdge[] = edgeEntries.map((e, i) => ({ id: i, + nodeId: '', // assigned by assignNodeIds once endpoint nodeIds exist + revision: buildRevision, sourceId: e.sourceId, targetId: e.targetId, kind: 'import', @@ -371,10 +412,42 @@ export async function assembleGraph( .sort((a, b) => a.sourceId - b.sourceId || a.targetId - b.targetId) let nextEdgeId = edges.length for (const c of callEntries) { - edges.push({ id: nextEdgeId++, sourceId: c.sourceId, targetId: c.targetId, kind: 'calls', weight: c.weight }) + edges.push({ id: nextEdgeId++, nodeId: '', revision: buildRevision, sourceId: c.sourceId, targetId: c.targetId, kind: 'calls', weight: c.weight }) } console.error(`[build] call edges done: ${callEntries.length} call edges, ${edges.length} total edges`) + // ---- Inheritance edges (class → superclass / interface) ------------------- + // Same resolution as calls: the enclosing symbol at the heritage line is the + // subtype; the supertype name resolves to a symbol in this file or an import. + console.error(`[build] building inheritance edges...`) + const inheritWeights = new Map() + for (const file of files) { + const inherits = rawInheritsByFile.get(file.id) ?? [] + if (inherits.length === 0) continue + const fileSymbols = symbolsByFile.get(file.id) ?? [] + for (const inh of inherits) { + const subtype = enclosing(fileSymbols, inh.line) + if (!subtype) continue + const supertype = resolveCallee(inh.supertype, file.id) + if (!supertype || supertype.id === subtype.id) continue + const key = `${subtype.id}->${supertype.id}:${inh.relation}` + const prev = inheritWeights.get(key) + if (prev) prev.weight += 1 + else inheritWeights.set(key, { weight: 1, relation: inh.relation }) + } + } + const inheritEntries = [...inheritWeights.entries()] + .map(([key, v]) => { + const [pair] = key.split(':') as [string] + const [sourceId, targetId] = pair.split('->').map(Number) as [number, number] + return { sourceId, targetId, weight: v.weight, relation: v.relation } + }) + .sort((a, b) => a.sourceId - b.sourceId || a.targetId - b.targetId || a.relation.localeCompare(b.relation)) + for (const c of inheritEntries) { + edges.push({ id: nextEdgeId++, nodeId: '', revision: buildRevision, sourceId: c.sourceId, targetId: c.targetId, kind: c.relation, weight: c.weight }) + } + console.error(`[build] inheritance edges done: ${inheritEntries.length} edges, ${edges.length} total edges`) + // Build adjacency index for O(1) import-edge neighbor lookups. const adjacency = new Map() for (const f of files) adjacency.set(f.id, { in: [], out: [] }) @@ -384,8 +457,13 @@ export async function assembleGraph( adjacency.get(e.targetId)!.in.push(e) } + // Assign stable, root-qualified string ids to every node now that ranges and + // numeric ids exist. The rootId is path-derived (stable across restarts). + assignNodeIds(rootIdFor(absRoot), files, edges, symbolsByFile) + console.error(`[build] done: ${files.length} files, ${symbols.length} symbols, ${edges.length} edges`) return { + revision: buildRevision, rootPath: absRoot, name: path.basename(absRoot), files, @@ -402,6 +480,9 @@ export interface IndexChanges { added: string[] removed: string[] changed: string[] + /** nodeIds whose revision advanced to the new graphRevision this reindex — the + * patch set for an incremental-update feed. Empty on a no-op reindex. */ + changedNodeIds: string[] } // Persistent-cache schema version. The LMDB cache stores parse results (keyed by @@ -411,8 +492,11 @@ export interface IndexChanges { // would be served verbatim, silently dropping imports/edges or aliasing symbols // to the wrong file. BUMP THIS whenever parse output or id allocation changes; // the Indexer wipes a cache whose stored version doesn't match. -export const CACHE_VERSION = 2 +// v3: symbols gained UTF-16 nameRange/fullRange — older cached parse results +// predate those fields and must be reparsed. +export const CACHE_VERSION = 3 const CACHE_VERSION_KEY = 'meta:cacheVersion' +const GRAPH_REVISION_KEY = 'meta:graphRevision' function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { if (a.length !== b.length) return false @@ -439,6 +523,10 @@ export class Indexer { readonly absRoot: string private readonly dbPath?: string private store?: Store + /** Monotonic build counter; loaded from store on first build, bumped per reindex. */ + private graphRevision = 0 + /** Per-node change memory across reindexes (in-memory; rebuilt each session). */ + private readonly revState: RevisionState = newRevisionState() constructor( private readonly rootPath: string, @@ -454,6 +542,9 @@ export class Indexer { const { LmdbStore } = await import('./stores/lmdb.js') this.store = new LmdbStore(this.dbPath) this.ensureCacheVersion(this.store) + // Resume the global revision counter so it stays monotonic across restarts. + const rev = this.store.get(GRAPH_REVISION_KEY) + if (rev && rev.length === 4) this.graphRevision = decodeU32(rev) } return (await this.run()).graph } @@ -515,7 +606,7 @@ export class Indexer { if (collision) this.idByPath.clear() } - const changes: IndexChanges = { added: [], removed: [], changed: [] } + const changes: IndexChanges = { added: [], removed: [], changed: [], changedNodeIds: [] } const nextSha = new Map() for (const sf of scanned) { nextSha.set(sf.relPath, sf.sha) @@ -540,7 +631,12 @@ export class Indexer { parseTimeoutMs: this.opts.parseTimeoutMs, }) - // Persist file IDs + SHA-512 change tracker to store. + // Bump the global revision once per reindex, then stamp per-node revisions and + // collect the changed set (the patch list for graph/updated). + this.graphRevision += 1 + changes.changedNodeIds = assignRevisions(graph, this.revState, this.graphRevision) + + // Persist file IDs + SHA-512 change tracker + revision counter to store. if (this.store) { for (const sf of scanned) { this.store.set(`id:${sf.relPath}`, encodeU32(graph.fileIdByPath.get(sf.relPath) ?? 0)) @@ -548,6 +644,7 @@ export class Indexer { for (const p of changes.removed) { this.store.delete(`id:${p}`) } + this.store.set(GRAPH_REVISION_KEY, encodeU32(this.graphRevision)) this.store.sync() } diff --git a/core/src/cancellation.ts b/core/src/cancellation.ts new file mode 100644 index 0000000..4be4a2f --- /dev/null +++ b/core/src/cancellation.ts @@ -0,0 +1,74 @@ +// Cancellation primitives. A caller fires and abandons requests constantly — a +// query for a line the user already scrolled past should be dropped, not executed. +// This mirrors the widely-used CancellationToken shape so an editor's token can be +// wired straight through to a cancel with no adaptation. + +export class CancellationError extends Error { + constructor(message = 'Operation cancelled') { + super(message) + this.name = 'CancellationError' + } +} + +export interface Disposable { + dispose(): void +} + +export interface CancellationToken { + readonly isCancellationRequested: boolean + /** Register a listener. Fires immediately (synchronously) if already cancelled. */ + onCancellationRequested(listener: () => void): Disposable + /** Throw {@link CancellationError} if cancellation has been requested. */ + throwIfCancelled(): void +} + +class Token implements CancellationToken { + private cancelled = false + private readonly listeners = new Set<() => void>() + + get isCancellationRequested(): boolean { + return this.cancelled + } + + onCancellationRequested(listener: () => void): Disposable { + if (this.cancelled) { + listener() + return { dispose() {} } + } + this.listeners.add(listener) + return { dispose: () => this.listeners.delete(listener) } + } + + throwIfCancelled(): void { + if (this.cancelled) throw new CancellationError() + } + + fire(): void { + if (this.cancelled) return + this.cancelled = true + for (const l of this.listeners) { + try { + l() + } catch { + // A misbehaving listener must not block the others or the canceller. + } + } + this.listeners.clear() + } +} + +/** Owns a token and the sole authority to cancel it (VS Code's model). */ +export class CancellationTokenSource { + private readonly _token = new Token() + + get token(): CancellationToken { + return this._token + } + + cancel(): void { + this._token.fire() + } +} + +/** A token that is never cancelled — a convenient default for callers with none. */ +export const NONE: CancellationToken = new CancellationTokenSource().token diff --git a/core/src/index.ts b/core/src/index.ts index 90698f0..8875f5f 100644 --- a/core/src/index.ts +++ b/core/src/index.ts @@ -68,3 +68,15 @@ export { type FindSymbolOptions, type DependencyHit, } from './query.js' +export { searchContent, type SearchContentOptions } from './search.js' +export { + traceCalls, + type TraceCallsOptions, + type TraceDirection, +} from './trace.js' +export { + rankSymbols, + tokenize, + type RankedSymbolHit, + type Bm25Options, +} from './bm25.js' diff --git a/core/src/lock.ts b/core/src/lock.ts new file mode 100644 index 0000000..e71d81c --- /dev/null +++ b/core/src/lock.ts @@ -0,0 +1,72 @@ +// Reader/writer lock. The graph is immutable between reindexes, so the model is: +// unlimited concurrent reads, an exclusive write held only during a reindex. No +// real parallelism is needed — just correct mutual exclusion so a caller firing +// many queries at once never observes a half-rebuilt graph. +// +// Fairness is FIFO: waiters are granted in arrival order, with reads batching until +// the next waiter is a writer. That single rule prevents BOTH failure modes — a +// stream of reads can't starve a queued reindex (it waits behind the writer at the +// head), and a stream of writes can't starve reads. This is what lets "two +// concurrent reads complete while a reindex is queued behind them" hold. + +type Waiter = { write: boolean; grant: () => void } + +export class RwLock { + private readers = 0 + private writer = false + private readonly queue: Waiter[] = [] + + /** Run `fn` under a shared read lock. Concurrent with other readers. */ + async read(fn: () => T | Promise): Promise { + await this.acquire(false) + try { + return await fn() + } finally { + this.readers-- + this.dispatch() + } + } + + /** Run `fn` under an exclusive write lock. No reader or writer overlaps it. */ + async write(fn: () => T | Promise): Promise { + await this.acquire(true) + try { + return await fn() + } finally { + this.writer = false + this.dispatch() + } + } + + /** Snapshot of lock state, for diagnostics/tests. */ + get state(): { readers: number; writer: boolean; waiting: number } { + return { readers: this.readers, writer: this.writer, waiting: this.queue.length } + } + + private acquire(write: boolean): Promise { + return new Promise((grant) => { + this.queue.push({ write, grant }) + this.dispatch() + }) + } + + private dispatch(): void { + while (this.queue.length > 0) { + const next = this.queue[0]! + if (next.write) { + // A writer runs only once every reader has drained and no writer holds. + if (this.readers > 0 || this.writer) break + this.queue.shift() + this.writer = true + next.grant() + break // exclusive: nothing else may proceed + } + // Reader at the head: cannot proceed while a writer holds the lock. + if (this.writer) break + this.queue.shift() + this.readers++ + next.grant() + // Loop on to batch further consecutive readers; stops at a queued writer. + } + } +} diff --git a/core/src/nodeid.ts b/core/src/nodeid.ts new file mode 100644 index 0000000..eb22858 --- /dev/null +++ b/core/src/nodeid.ts @@ -0,0 +1,153 @@ +// Stable, root-qualified string identity for graph nodes. +// +// The numeric `id` on files/symbols/edges is an in-memory adjacency index — it is +// reassigned every build and must never be persisted as cross-reindex identity. +// The STRING `nodeId` is the external identity that external consumers (editors, +// caches, incremental-update feeds) rely on. It is: +// +// • Structural, not content-based — editing a function body does not change it +// (content-hash ids were rejected: every keystroke would invalidate them). +// • Root-qualified — prefixed by a per-root handle so two workspace roots in one +// process never collide. +// • Deterministic — same repo bytes → same ids, so an unchanged reindex is a no-op +// for every consumer keyed on nodeId. +// +// Shape: :: (file) +// ::# (symbol; dotted = enclosing chain) +// ::#@ (overloads / duplicate paths) +// :-> (edge) +// +// A rename is delete+create (a new nodeId), which is correct: a renamed symbol is a +// different node. + +import { sha512 } from './hash.js' +import type { CodeEdge, CodeFile, CodeSymbol } from './types.js' + +/** + * Stable per-workspace-root handle: 12 hex chars of the normalized absolute root + * path's hash. Path-derived, so it survives process restarts (unlike a process- + * assigned counter) — which is what lets external caches key on nodeId across + * restarts. Namespaces every node id by root. + */ +export function rootIdFor(absRootPath: string): string { + return sha512(absRootPath).slice(0, 12) +} + +/** `::` — the identity of a file node. */ +export function fileNodeId(rootId: string, relPath: string): string { + return `${rootId}::${relPath}` +} + +/** Minimal shape needed to compute a structural id — satisfied by CodeSymbol. */ +interface RangedNamed { + name: string + fullRange: readonly [number, number] +} + +/** + * Compute the structural nodeId of every symbol in ONE file, returned parallel to + * `fileSymbols`. The dotted prefix is the chain of enclosing symbols, determined by + * UTF-16 `fullRange` containment (smallest strict container wins). Symbols that + * resolve to the same structural path — overloads, or genuine duplicate names — get + * a stable `@ordinal` by source order; a unique path gets no suffix. + * + * Pure and deterministic: depends only on names and ranges, so it is directly + * unit-testable with synthetic symbols and identical across unchanged reindexes. + */ +export function computeSymbolNodeIds( + rootId: string, + relPath: string, + fileSymbols: readonly RangedNamed[], +): string[] { + const n = fileSymbols.length + // container[i] = index of the smallest symbol that STRICTLY encloses i, or -1. + const container: number[] = new Array(n).fill(-1) + for (let i = 0; i < n; i++) { + const a = fileSymbols[i]! + let bestJ = -1 + let bestSpan = Number.POSITIVE_INFINITY + for (let j = 0; j < n; j++) { + if (j === i) continue + const b = fileSymbols[j]! + const enclosesInclusive = b.fullRange[0] <= a.fullRange[0] && a.fullRange[1] <= b.fullRange[1] + const strictlyLarger = b.fullRange[0] < a.fullRange[0] || a.fullRange[1] < b.fullRange[1] + if (!enclosesInclusive || !strictlyLarger) continue + const span = b.fullRange[1] - b.fullRange[0] + if (span < bestSpan) { + bestSpan = span + bestJ = j + } + } + container[i] = bestJ + } + + // Dotted structural path per symbol, walking container links outward. + const pathOf: string[] = new Array(n) + for (let i = 0; i < n; i++) { + const parts: string[] = [] + let cur = i + const guard = new Set() + while (cur !== -1 && !guard.has(cur)) { + guard.add(cur) // defensive: never loop on a degenerate containment cycle + parts.push(fileSymbols[cur]!.name) + cur = container[cur]! + } + parts.reverse() + pathOf[i] = parts.join('.') + } + + // Group identical paths; disambiguate with a stable source-order ordinal. + const groups = new Map() + for (let i = 0; i < n; i++) { + const list = groups.get(pathOf[i]!) + if (list) list.push(i) + else groups.set(pathOf[i]!, [i]) + } + const ids: string[] = new Array(n) + for (const [p, idxs] of groups) { + if (idxs.length === 1) { + ids[idxs[0]!] = `${rootId}::${relPath}#${p}` + } else { + idxs.forEach((idx, ord) => { + ids[idx] = `${rootId}::${relPath}#${p}@${ord}` + }) + } + } + return ids +} + +/** + * Assign `nodeId` in place to every file, symbol, and edge of a graph. Files and + * symbols get structural ids; edges get `:->` (import + * endpoints are files, call endpoints are symbols). Runs once at the end of graph + * assembly, after ranges and numeric ids exist. + */ +export function assignNodeIds( + rootId: string, + files: readonly CodeFile[], + edges: readonly CodeEdge[], + symbolsByFile: ReadonlyMap, +): void { + const fileNodeById = new Map() + for (const f of files) { + f.nodeId = fileNodeId(rootId, f.path) + fileNodeById.set(f.id, f.nodeId) + } + + const symbolNodeById = new Map() + for (const f of files) { + const syms = symbolsByFile.get(f.id) ?? [] + const ids = computeSymbolNodeIds(rootId, f.path, syms) + for (let i = 0; i < syms.length; i++) { + const s = syms[i]! + s.nodeId = ids[i]! + symbolNodeById.set(s.id, s.nodeId) + } + } + + for (const e of edges) { + const src = e.kind === 'import' ? fileNodeById.get(e.sourceId) : symbolNodeById.get(e.sourceId) + const tgt = e.kind === 'import' ? fileNodeById.get(e.targetId) : symbolNodeById.get(e.targetId) + e.nodeId = `${e.kind}:${src ?? e.sourceId}->${tgt ?? e.targetId}` + } +} diff --git a/core/src/parse/grammars/index.ts b/core/src/parse/grammars/index.ts index d42a212..b011a6b 100644 --- a/core/src/parse/grammars/index.ts +++ b/core/src/parse/grammars/index.ts @@ -41,6 +41,13 @@ export interface GrammarConfig { symbolQuery: string importQuery: string | null callQuery?: string + /** + * Optional query capturing inheritance. Each match should yield a + * `@extends` and/or `@implements` capture naming a supertype; the enclosing + * class/interface (found by line) is the subtype. Resolved to symbol edges in + * build, mirroring call resolution. + */ + inheritQuery?: string importSpecifier: (node: Node) => string keep: (def: Node, name: string) => boolean exported: (def: Node, name: string) => boolean diff --git a/core/src/parse/grammars/java.ts b/core/src/parse/grammars/java.ts index 94ed70a..b621fe9 100644 --- a/core/src/parse/grammars/java.ts +++ b/core/src/parse/grammars/java.ts @@ -4,7 +4,7 @@ const JAVA_SYMBOLS = ` (class_declaration name: (identifier) @name) @def.class (interface_declaration name: (identifier) @name) @def.interface (enum_declaration name: (identifier) @name) @def.type -(method_declaration name: (identifier) @name) @def.function +(method_declaration name: (identifier) @name) @def.method ` const JAVA_IMPORTS = ` (import_declaration (scoped_identifier) @path) @@ -13,6 +13,13 @@ const JAVA_CALLS = ` (method_invocation name: (identifier) @callee) (method_invocation object: (identifier) (identifier) @callee) ` +// Java heritage: `extends Super` (class), `implements I, J` (class), +// `extends A, B` (interface). Capture the bare type identifiers. +const JAVA_INHERIT = ` +(superclass (type_identifier) @extends) +(super_interfaces (type_list (type_identifier) @implements)) +(extends_interfaces (type_list (type_identifier) @extends)) +` function firstLine(text: string): string { const nl = text.indexOf('\n') @@ -33,6 +40,7 @@ export const java: GrammarConfig = { symbolQuery: JAVA_SYMBOLS, importQuery: JAVA_IMPORTS, callQuery: JAVA_CALLS, + inheritQuery: JAVA_INHERIT, keep: () => true, exported: (def) => /\bpublic\b/.test(firstLine(def.text)), importSpecifier: (n) => n.text, diff --git a/core/src/parse/grammars/javascript.ts b/core/src/parse/grammars/javascript.ts index 9a03582..23ec021 100644 --- a/core/src/parse/grammars/javascript.ts +++ b/core/src/parse/grammars/javascript.ts @@ -9,6 +9,7 @@ const JS_SYMBOLS = ` (class_declaration name: (identifier) @name) @def.class (lexical_declaration (variable_declarator name: (identifier) @name)) @def.const (variable_declaration (variable_declarator name: (identifier) @name)) @def.const +(method_definition name: (property_identifier) @name) @def.method ` const JS_IMPORTS = ` (import_statement source: (string) @source) @@ -20,6 +21,12 @@ const JS_CALLS = ` (call_expression function: (member_expression property: (property_identifier) @callee)) (new_expression constructor: (identifier) @callee) ` +// JS class heritage is `extends ` directly (no extends_clause node, +// no interfaces). +const JS_INHERIT = ` +(class_heritage (identifier) @extends) +(class_heritage (member_expression property: (property_identifier) @extends)) +` function exported(def: import('web-tree-sitter').Node): boolean { let n: import('web-tree-sitter').Node | null = def @@ -38,6 +45,22 @@ function topLevel(def: import('web-tree-sitter').Node): boolean { return false } +function enclosingClass(def: import('web-tree-sitter').Node): import('web-tree-sitter').Node | null { + let n: import('web-tree-sitter').Node | null = def.parent + for (let i = 0; i < 4 && n; i++) { + if (n.type === 'class_declaration' || n.type === 'class') return n + n = n.parent + } + return null +} + +/** Keep a method only when its owning class is kept (top-level or exported). */ +function keptMember(def: import('web-tree-sitter').Node): boolean { + const cls = enclosingClass(def) + if (!cls) return false + return topLevel(cls) || exported(cls) +} + // Same TS/JS import resolution as typescript const TSJS_EXTS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs'] @@ -99,8 +122,16 @@ export const javascript: GrammarConfig = { symbolQuery: JS_SYMBOLS, importQuery: JS_IMPORTS, callQuery: JS_CALLS, - keep: (def) => topLevel(def) || exported(def), - exported: (def) => exported(def), + inheritQuery: JS_INHERIT, + keep: (def) => + def.type === 'method_definition' ? keptMember(def) : topLevel(def) || exported(def), + exported: (def) => { + if (def.type === 'method_definition') { + const cls = enclosingClass(def) + return cls ? exported(cls) : false + } + return exported(def) + }, importSpecifier: (n) => { const s = n.text if (s.length >= 2) { diff --git a/core/src/parse/grammars/python.ts b/core/src/parse/grammars/python.ts index 04b830c..5d59c8f 100644 --- a/core/src/parse/grammars/python.ts +++ b/core/src/parse/grammars/python.ts @@ -17,6 +17,20 @@ const PY_CALLS = ` (call function: (identifier) @callee) (call function: (attribute attribute: (identifier) @callee)) ` +// `class Foo(Base, mixins.Other):` — base classes live in the superclasses +// argument list. Captured as `extends` (Python has no separate implements). +const PY_INHERIT = ` +(class_definition superclasses: (argument_list (identifier) @extends)) +(class_definition superclasses: (argument_list (attribute attribute: (identifier) @extends))) +` + +/** True when `def` is a method: a function_definition whose nearest block parent + * is the body of a class_definition. */ +function isMethod(def: import('web-tree-sitter').Node): boolean { + const block = def.parent + if (!block || block.type !== 'block') return false + return block.parent?.type === 'class_definition' +} function resolvePython(fromRel: string, spec: string, bySet: Set): string | null { const trimmed = spec.trim() @@ -45,8 +59,13 @@ export const python: GrammarConfig = { symbolQuery: PY_SYMBOLS, importQuery: PY_IMPORTS, callQuery: PY_CALLS, - keep: (def) => def.parent?.type === 'module', - exported: (def, name) => def.parent?.type === 'module' && !name.startsWith('_'), + inheritQuery: PY_INHERIT, + // Keep top-level defs and class methods (build promotes methods' kind by + // containment). Methods of underscore-private classes still count as members. + keep: (def) => def.parent?.type === 'module' || isMethod(def), + // Public surface: a top-level non-underscore name, or a non-underscore method. + exported: (def, name) => + (def.parent?.type === 'module' || isMethod(def)) && !name.startsWith('_'), importSpecifier: (n) => n.text, resolveImport: resolvePython, } diff --git a/core/src/parse/grammars/typescript.ts b/core/src/parse/grammars/typescript.ts index 156764e..17e9d77 100644 --- a/core/src/parse/grammars/typescript.ts +++ b/core/src/parse/grammars/typescript.ts @@ -13,6 +13,7 @@ const TS_SYMBOLS = ` (enum_declaration name: (identifier) @name) @def.type (lexical_declaration (variable_declarator name: (identifier) @name)) @def.const (variable_declaration (variable_declarator name: (identifier) @name)) @def.const +(method_definition name: (property_identifier) @name) @def.method ` const TS_IMPORTS = ` (import_statement source: (string) @source) @@ -24,6 +25,16 @@ const TS_CALLS = ` (call_expression function: (member_expression property: (property_identifier) @callee)) (new_expression constructor: (identifier) @callee) ` +// Class/interface heritage → `extends`/`implements` supertype names. Captures the +// bare type identifier (generics unwrap to their head, e.g. Foo → Foo). +const TS_INHERIT = ` +(extends_clause (identifier) @extends) +(extends_clause (member_expression property: (property_identifier) @extends)) +(implements_clause (type_identifier) @implements) +(implements_clause (generic_type (type_identifier) @implements)) +(extends_type_clause (type_identifier) @extends) +(extends_type_clause (generic_type (type_identifier) @extends)) +` function exported(def: import('web-tree-sitter').Node): boolean { let n: import('web-tree-sitter').Node | null = def @@ -42,6 +53,27 @@ function topLevel(def: import('web-tree-sitter').Node): boolean { return false } +/** The enclosing class/interface declaration of a node, if any (walks out + * through class_body). Used to keep methods only of kept classes and to mark a + * method's public surface from its class. */ +function enclosingClass(def: import('web-tree-sitter').Node): import('web-tree-sitter').Node | null { + let n: import('web-tree-sitter').Node | null = def.parent + for (let i = 0; i < 4 && n; i++) { + if (n.type === 'class_declaration' || n.type === 'abstract_class_declaration' || n.type === 'class') return n + n = n.parent + } + return null +} + +/** Keep a method only if its owning class is itself kept (top-level or exported) + * — so we add the members that matter without flooding the graph with the + * methods of throwaway local classes. */ +function keptMember(def: import('web-tree-sitter').Node): boolean { + const cls = enclosingClass(def) + if (!cls) return false + return topLevel(cls) || exported(cls) +} + // --------------------------------------------------------------------------- // Import resolution // --------------------------------------------------------------------------- @@ -106,8 +138,18 @@ export const typescript: GrammarConfig = { symbolQuery: TS_SYMBOLS, importQuery: TS_IMPORTS, callQuery: TS_CALLS, - keep: (def) => topLevel(def) || exported(def), - exported: (def) => exported(def), + inheritQuery: TS_INHERIT, + keep: (def) => + def.type === 'method_definition' ? keptMember(def) : topLevel(def) || exported(def), + // A method's public surface follows its class; other symbols follow their own + // export marker. + exported: (def) => { + if (def.type === 'method_definition') { + const cls = enclosingClass(def) + return cls ? exported(cls) : false + } + return exported(def) + }, importSpecifier: (n) => { const s = n.text if (s.length >= 2) { diff --git a/core/src/parse/index.ts b/core/src/parse/index.ts index c4c81c7..ec80339 100644 --- a/core/src/parse/index.ts +++ b/core/src/parse/index.ts @@ -27,14 +27,14 @@ function signatureOf(def: Node): string { * (default no limit); on timeout the file silently yields an empty result. */ export async function parseFile(relPath: string, content: string, parseTimeoutMs?: number): Promise { const grammar = grammarForFile(relPath) - if (!grammar) return { symbols: [], imports: [], calls: [] } + if (!grammar) return { symbols: [], imports: [], calls: [], inherits: [] } const config: GrammarConfig | undefined = GRAMMARS[grammar] - if (!config) return { symbols: [], imports: [], calls: [] } + if (!config) return { symbols: [], imports: [], calls: [], inherits: [] } let root try { root = await parseSource(grammar, content, parseTimeoutMs) } catch { - return { symbols: [], imports: [], calls: [] } + return { symbols: [], imports: [], calls: [], inherits: [] } } const queries = getOrCompileQueries(grammar, config) @@ -65,6 +65,11 @@ export async function parseFile(relPath: string, content: string, parseTimeoutMs signature: signatureOf(defNode), startLine: defNode.startPosition.row + 1, endLine: defNode.endPosition.row + 1, + // web-tree-sitter node indices are UTF-16 code units into the source + // string (verified: source.slice(startIndex, endIndex) === node.text), + // which is exactly what VS Code Positions consume. Store as-is. + nameRange: [nameNode.startIndex, nameNode.endIndex], + fullRange: [defNode.startIndex, defNode.endIndex], exported: config.exported(defNode, name), }) } @@ -105,5 +110,22 @@ export async function parseFile(relPath: string, content: string, parseTimeoutMs } } catch (e) { console.error(`[parse] call query failed for ${relPath}: ${e}`) } - return { symbols, imports, calls } + let inherits: ParseResult['inherits'] = [] + try { + if (queries.inheritQuery) { + const extracted: ParseResult['inherits'] = [] + for (const match of queries.inheritQuery.matches(root)) { + for (const cap of match.captures) { + if (cap.name !== 'extends' && cap.name !== 'implements') continue + const supertype = cap.node.text.trim() + if (supertype.length > 0) { + extracted.push({ supertype, relation: cap.name, line: cap.node.startPosition.row + 1 }) + } + } + } + inherits = extracted + } + } catch (e) { console.error(`[parse] inherit query failed for ${relPath}: ${e}`) } + + return { symbols, imports, calls, inherits } } diff --git a/core/src/parse/treesitter.ts b/core/src/parse/treesitter.ts index 46e597b..96be913 100644 --- a/core/src/parse/treesitter.ts +++ b/core/src/parse/treesitter.ts @@ -188,18 +188,36 @@ export interface CompiledQueries { symbolQuery: Query importQuery: Query | null callQuery: Query | null + inheritQuery: Query | null } const queryCache = new Map() -export function getOrCompileQueries(id: GrammarId, config: { symbolQuery: string; importQuery: string | null; callQuery?: string }): CompiledQueries { - const key = `${id}::${config.symbolQuery}::${config.importQuery}::${config.callQuery ?? ''}` +/** Compile an optional query, tolerating failures: a query that references a + * node type this grammar version doesn't have compiles to `null` (logged once) + * instead of throwing and disabling all extraction for the grammar. */ +function compileOptional(id: GrammarId, source: string | null | undefined): Query | null { + if (!source) return null + try { + return compileQuery(id, source) + } catch (e) { + console.error(`[parse] query compile failed for ${id} (skipped): ${e}`) + return null + } +} + +export function getOrCompileQueries( + id: GrammarId, + config: { symbolQuery: string; importQuery: string | null; callQuery?: string; inheritQuery?: string }, +): CompiledQueries { + const key = `${id}::${config.symbolQuery}::${config.importQuery}::${config.callQuery ?? ''}::${config.inheritQuery ?? ''}` let cached = queryCache.get(key) if (!cached) { cached = { symbolQuery: compileQuery(id, config.symbolQuery), - importQuery: config.importQuery ? compileQuery(id, config.importQuery) : null, - callQuery: config.callQuery ? compileQuery(id, config.callQuery) : null, + importQuery: compileOptional(id, config.importQuery), + callQuery: compileOptional(id, config.callQuery), + inheritQuery: compileOptional(id, config.inheritQuery), } queryCache.set(key, cached) } diff --git a/core/src/position.ts b/core/src/position.ts new file mode 100644 index 0000000..d5fd916 --- /dev/null +++ b/core/src/position.ts @@ -0,0 +1,62 @@ +// The single place a stored source offset becomes an editor Position. +// +// OpenVisio stores every range (nameRange, fullRange) as a pair of UTF-16 +// code-unit offsets into the file's source. Two empirically verified facts make +// that the correct unit — and make this conversion pure arithmetic with no +// encoding step: +// +// 1. The parser is web-tree-sitter (WASM). Given a JS string, its node offsets +// (startIndex/endIndex) and row/column are UTF-16 code units — for every +// node, source.slice(node.startIndex, node.endIndex) === node.text, holding +// for emoji, CJK, and combining marks alike. +// 2. VS Code's Position.character and TextDocument.positionAt(offset) are ALSO +// UTF-16 code units. +// +// So a stored offset maps to an editor Position with one line lookup and a +// subtraction — no UTF-8 byte math. Storing BYTE offsets instead would force a +// byte→UTF-16 conversion here and reintroduce the exact column-drift bug that any +// file with an emoji/CJK/accented character above the target column otherwise +// produces. Keep every offset in this codebase UTF-16, and keep this module the +// ONLY converter. + +/** 0-based line + 0-based UTF-16 character. Shape-compatible with vscode.Position. */ +export interface Position { + line: number + character: number +} + +/** + * Convert a whole-file UTF-16 offset to a 0-based {line, character}. + * `lineStarts[i]` is the UTF-16 offset where line `i` begins; `lineStarts[0]` is + * always 0 (see {@link computeLineStarts}). O(log n) binary search for the hot + * cursor / annotation lookup paths. Negative offsets clamp to the start; offsets + * past EOF resolve on the last line. + */ +export function offsetToPosition(lineStarts: readonly number[], offset: number): Position { + const n = lineStarts.length + if (n === 0) return { line: 0, character: Math.max(0, offset) } + if (offset <= 0) return { line: 0, character: 0 } + // Greatest i such that lineStarts[i] <= offset. + let lo = 0 + let hi = n - 1 + while (lo < hi) { + const mid = (lo + hi + 1) >> 1 + if ((lineStarts[mid] as number) <= offset) lo = mid + else hi = mid - 1 + } + return { line: lo, character: offset - (lineStarts[lo] as number) } +} + +/** + * UTF-16 offsets at which each line starts. `[0]` is always 0; one entry per + * line. A `\n` (including the `\n` of a `\r\n`) ends the current line and the + * next line starts after it — matching how editors and tree-sitter count rows. + * A trailing `\n` yields a final entry for the empty last line, as VS Code models it. + */ +export function computeLineStarts(source: string): number[] { + const starts = [0] + for (let i = 0; i < source.length; i++) { + if (source.charCodeAt(i) === 10 /* \n */) starts.push(i + 1) + } + return starts +} diff --git a/core/src/revision.ts b/core/src/revision.ts new file mode 100644 index 0000000..91d58cf --- /dev/null +++ b/core/src/revision.ts @@ -0,0 +1,84 @@ +// Revision tracking — what makes incremental reindex cheap for consumers. +// +// Two numbers: +// • graphRevision — a global monotonic counter bumped once per completed +// reindex. Returned on every response so a consumer can detect a stale view. +// • node.revision — the graphRevision at which THAT node last changed. Unchanged +// nodes carry their prior revision forward, so an incremental-update feed can +// ship only the genuinely-changed node ids and a consumer patches instead of +// refetching. +// +// "Changed" is decided by a content signature per node: an edit that leaves a +// node's structural surface (name, kind, signature, ranges, edge endpoints/weight) +// identical does not bump its revision — consumers render structure, not bodies. + +import type { CodeEdge, CodeFile, CodeGraph, CodeSymbol } from './types.js' + +/** + * Cross-reindex memory: for each nodeId, the signature it had and the revision at + * which it last changed. Held by the long-lived Indexer; compared on each reindex. + */ +export interface RevisionState { + sig: Map + rev: Map +} + +export function newRevisionState(): RevisionState { + return { sig: new Map(), rev: new Map() } +} + +// Signatures capture exactly the surface the UI renders — deliberately NOT the +// body — so a body-only edit bumps the file's revision (its sha changed) but not +// the symbol's. +function fileSig(f: CodeFile): string { + return `f|${f.sha}` +} +function symbolSig(s: CodeSymbol): string { + return `s|${s.kind}|${s.name}|${s.signature}|${s.fullRange[0]},${s.fullRange[1]}|${s.nameRange[0]},${s.nameRange[1]}|${s.exported ? 1 : 0}` +} +function edgeSig(e: CodeEdge): string { + // nodeId already encodes kind + both endpoints; weight is the only other surface. + return `e|${e.weight}` +} + +/** + * Stamp `revision` on every node of `graph` and return the nodeIds that changed at + * `newRevision`. A node changes if it is new or its signature differs from the + * prior build; otherwise it carries its previous revision. `graph.revision` is set + * to `newRevision`. Mutates `state` so the next reindex compares against this build; + * ids that vanished (rename/delete) are pruned from state. + */ +export function assignRevisions( + graph: CodeGraph, + state: RevisionState, + newRevision: number, +): string[] { + const changed: string[] = [] + const live = new Set() + + const stamp = (nodeId: string, sig: string, set: (rev: number) => void): void => { + live.add(nodeId) + if (state.sig.get(nodeId) === sig) { + set(state.rev.get(nodeId) ?? newRevision) + return + } + state.sig.set(nodeId, sig) + state.rev.set(nodeId, newRevision) + set(newRevision) + changed.push(nodeId) + } + + for (const f of graph.files) stamp(f.nodeId, fileSig(f), (r) => (f.revision = r)) + for (const s of graph.symbols) stamp(s.nodeId, symbolSig(s), (r) => (s.revision = r)) + for (const e of graph.edges) stamp(e.nodeId, edgeSig(e), (r) => (e.revision = r)) + + for (const id of [...state.sig.keys()]) { + if (!live.has(id)) { + state.sig.delete(id) + state.rev.delete(id) + } + } + + graph.revision = newRevision + return changed +} diff --git a/core/src/roots.ts b/core/src/roots.ts new file mode 100644 index 0000000..e062c58 --- /dev/null +++ b/core/src/roots.ts @@ -0,0 +1,84 @@ +// Multi-root registry — the set of workspace roots one process serves. +// +// One process serves many roots (lower memory than one process per folder): each +// root gets its own Indexer, and every operation is addressed by an explicit +// `rootId` — there is no implicit "current" root. Graphs are fully isolated: +// because nodeIds are root-qualified (see nodeid.ts), no id or edge can bleed +// between roots even though they share a process. Storage, when a cacheDir is +// given, is namespaced by rootId on disk. + +import * as path from 'node:path' +import { Indexer, type BuildOptions, type IndexChanges } from './build.js' +import { rootIdFor } from './nodeid.js' +import type { CodeGraph } from './types.js' + +export interface RegisteredRoot { + /** Stable, path-derived handle (see rootIdFor) — the key for every request. */ + rootId: string + /** Absolute, normalized root path. */ + rootPath: string +} + +export class RootRegistry { + private readonly roots = new Map() + + constructor( + private readonly opts: { cacheDir?: string; buildOptions?: BuildOptions } = {}, + ) {} + + /** The rootId a path resolves to, without registering it. */ + rootIdForPath(rootPath: string): string { + return rootIdFor(path.resolve(rootPath)) + } + + /** Register a workspace root. Idempotent — re-registering a path returns the same + * rootId and keeps the existing Indexer (and its warm cache). */ + register(rootPath: string): string { + const absRoot = path.resolve(rootPath) + const rootId = rootIdFor(absRoot) + if (!this.roots.has(rootId)) { + const dbPath = this.opts.cacheDir ? path.join(this.opts.cacheDir, rootId) : undefined + const indexer = new Indexer(absRoot, this.opts.buildOptions ?? {}, dbPath) + this.roots.set(rootId, { root: { rootId, rootPath: absRoot }, indexer }) + } + return rootId + } + + /** Deregister a root and release its Indexer/store. No-op if unknown. */ + unregister(rootId: string): void { + const entry = this.roots.get(rootId) + if (!entry) return + entry.indexer.close() + this.roots.delete(rootId) + } + + has(rootId: string): boolean { + return this.roots.has(rootId) + } + + list(): RegisteredRoot[] { + return [...this.roots.values()].map((e) => e.root) + } + + private require(rootId: string): Indexer { + const entry = this.roots.get(rootId) + if (!entry) throw new Error(`unknown root: ${rootId}`) + return entry.indexer + } + + /** Build (cold) the graph for one root. */ + build(rootId: string): Promise { + return this.require(rootId).build() + } + + /** Reindex one root, returning its graph and the change set (incl. changedNodeIds). */ + reindex(rootId: string): Promise<{ graph: CodeGraph; changes: IndexChanges }> { + return this.require(rootId).reindex() + } + + /** Release every root's Indexer. */ + closeAll(): void { + for (const e of this.roots.values()) e.indexer.close() + this.roots.clear() + } +} diff --git a/core/src/search.ts b/core/src/search.ts new file mode 100644 index 0000000..7ee0828 --- /dev/null +++ b/core/src/search.ts @@ -0,0 +1,251 @@ +// Content search — the grep replacement. The structural tools (find_symbol, +// get_neighborhood, get_dependents) answer "where is this symbol and how does it +// connect". They can't answer "where does this literal string / regex / TODO / +// error message / config key appear", which is why agents fall back to grep. +// +// This closes that gap WITHOUT leaving the graph: it searches the SAME curated +// file set OpenVisio already indexed (source only — no node_modules, no +// binaries, no lockfiles), reads each file on demand, and returns matches that +// are ranked by containing-file centrality, annotated with the enclosing symbol, +// and carry an exact `path:line` anchor + snippet. Token-budgeted like every +// other view. Deterministic: no LLM, byte-stable ordering. + +import { TokenBudget } from './budget.js' +import { computeCentrality, type Centrality } from './rank.js' +import type { ViewResult } from './skeleton.js' +import type { CodeFile, CodeGraph, CodeSymbol } from './types.js' +import * as fs from 'node:fs' +import * as path from 'node:path' + +// Reuse-friendly line cache, keyed by absolute path + mtime so --watch edits +// invalidate naturally (mirrors slice.ts, kept local to avoid coupling). +const fileCache = new Map() +const CACHE_MAX = 200 + +const NUL = String.fromCharCode(0) + +function readLines(absPath: string): string[] | null { + let stat: fs.Stats + try { + stat = fs.statSync(absPath) + } catch { + return null + } + const cached = fileCache.get(absPath) + if (cached && cached.mtimeMs === stat.mtimeMs) return cached.lines + let content: string + try { + content = fs.readFileSync(absPath, 'utf8') + } catch { + return null + } + // Skip anything that looks binary (a NUL byte). The indexed set already + // excludes known binaries, but this is a cheap belt-and-suspenders guard. + if (content.includes(NUL)) return null + const lines = content.split('\n') + if (fileCache.size >= CACHE_MAX) { + const first = fileCache.keys().next().value + if (first !== undefined) fileCache.delete(first) + } + fileCache.set(absPath, { mtimeMs: stat.mtimeMs, lines }) + return lines +} + +export interface SearchContentOptions { + /** The search text. Treated as a literal unless `regex` is true. */ + query: string + /** Interpret `query` as a JS regular expression instead of a literal. */ + regex?: boolean + /** Case-sensitive match (default false — case-insensitive like ripgrep -i). */ + caseSensitive?: boolean + /** + * Restrict to files whose repo-relative path matches this filter. A glob when + * it contains `*`/`?` (e.g. `*.ts`, `src/api/*`), else a case-insensitive + * substring (e.g. `components/`, `client`). + */ + pathFilter?: string + /** Max matching lines emitted per file before eliding the rest (default 5). */ + maxPerFile?: number + /** Hard cap on total matching lines collected before budgeting (default 60). */ + limit?: number + /** Token ceiling for the whole view (default 1500). */ + budgetTokens?: number + /** Reuse a precomputed centrality (else computed on the fly). */ + centrality?: Centrality +} + +/** One matching line, resolved to its file, enclosing symbol, and centrality. */ +interface Match { + file: CodeFile + line: number + text: string + enclosing: CodeSymbol | null + score: number +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +function compileQuery(opts: SearchContentOptions): RegExp | null { + const flags = opts.caseSensitive ? 'g' : 'gi' + const src = opts.regex ? opts.query : escapeRegExp(opts.query) + try { + return new RegExp(src, flags) + } catch { + return null + } +} + +/** Build a path predicate from `pathFilter`: glob when it has wildcards, else a + * case-insensitive substring match. */ +function pathMatcher(filter: string | undefined): (p: string) => boolean { + if (!filter) return () => true + if (/[*?]/.test(filter)) { + const rx = escapeRegExp(filter).replace(/\\\*/g, '.*').replace(/\\\?/g, '.') + let re: RegExp + try { + re = new RegExp(rx, 'i') + } catch { + return () => true + } + return (p) => re.test(p) + } + const needle = filter.toLowerCase() + return (p) => p.toLowerCase().includes(needle) +} + +/** Innermost symbol whose 1-based line range contains `line` (smallest span). */ +function enclosingSymbol(syms: CodeSymbol[] | undefined, line: number): CodeSymbol | null { + if (!syms) return null + let best: CodeSymbol | null = null + for (const s of syms) { + if (line < s.startLine || line > s.endLine) continue + if (!best || s.endLine - s.startLine < best.endLine - best.startLine) best = s + } + return best +} + +/** A snippet of the matching line, trimmed and capped so one long minified line + * can't blow the budget. */ +function snippet(text: string, max = 160): string { + const t = text.replace(/\t/g, ' ').trimEnd() + if (t.length <= max) return t.trimStart() + return t.trimStart().slice(0, max) + ' …' +} + +/** + * Search file contents across the indexed repo. Returns a ranked, anchored, + * token-budgeted view — the deterministic replacement for grep/rg. Matches are + * grouped by file (most import-central first), each line carrying an + * `@path:line` anchor and the enclosing symbol so the agent can drill straight + * in without a whole-file read. + */ +export function searchContent(graph: CodeGraph, opts: SearchContentOptions): ViewResult { + if (!opts.query) return { text: 'Provide a non-empty `query` to search for.', fileIds: [] } + const re = compileQuery(opts) + if (!re) return { text: `Invalid regex: ${opts.query}`, fileIds: [] } + + const centrality = opts.centrality ?? computeCentrality(graph) + const maxPerFile = opts.maxPerFile ?? 5 + const limit = opts.limit ?? 60 + const matchesPath = pathMatcher(opts.pathFilter) + + // Scan files most-central-first so, when we hit the limit, we keep the matches + // that matter. Stable tiebreak on path. + const files = graph.files + .filter((f) => matchesPath(f.path)) + .sort((a, b) => { + const sa = centrality.scoreByFile.get(a.id) ?? 0 + const sb = centrality.scoreByFile.get(b.id) ?? 0 + if (sb !== sa) return sb - sa + return a.path.localeCompare(b.path) + }) + + const byFile = new Map() + let totalMatches = 0 + let filesWithMatches = 0 + let truncated = false + + outer: for (const file of files) { + const lines = readLines(path.join(graph.rootPath, file.path)) + if (!lines) continue + const syms = graph.symbolsByFile.get(file.id) + const score = centrality.scoreByFile.get(file.id) ?? 0 + let fileHits: Match[] | null = null + for (let i = 0; i < lines.length; i++) { + re.lastIndex = 0 + if (!re.test(lines[i]!)) continue + if (!fileHits) { + fileHits = [] + byFile.set(file.id, fileHits) + filesWithMatches++ + } + fileHits.push({ + file, + line: i + 1, + text: lines[i]!, + enclosing: enclosingSymbol(syms, i + 1), + score, + }) + totalMatches++ + if (totalMatches >= limit) { + truncated = true + break outer + } + } + } + + const header = + `# search: ${opts.regex ? `/${opts.query}/` : `"${opts.query}"`}` + + (opts.pathFilter ? ` in ${opts.pathFilter}` : '') + + ` — ${totalMatches} match(es) in ${filesWithMatches} file(s)` + + (truncated ? ` (stopped at ${limit})` : '') + + '\n' + + if (totalMatches === 0) { + return { + text: + header + + '\nNo matches. Try a broader pattern, drop `path_filter`, or set `regex: true`. ' + + 'For a symbol by name use find_symbol; for who-calls-what use get_dependents.', + fileIds: [], + } + } + + const budget = new TokenBudget(opts.budgetTokens ?? 1500) + budget.add(header) + const out: string[] = [] + const fileIds: number[] = [] + + // byFile insertion order already follows centrality (files were pre-sorted). + for (const [fileId, hits] of byFile) { + const file = graph.filesById.get(fileId)! + const fileHeader = `\n${file.path} (${file.language}) — ${hits.length} hit(s)` + const firstHit = hits[0] + ? `\n ${hits[0].line}: ${snippet(hits[0].text)} @${file.path}:${hits[0].line}` + : '' + if (budget.wouldExceed(fileHeader + firstHit)) { + out.push(`\n… more matches omitted (budget reached). Narrow with path_filter or raise budget_tokens.`) + break + } + budget.add(fileHeader) + out.push(fileHeader) + fileIds.push(fileId) + let shown = 0 + for (const h of hits) { + if (shown >= maxPerFile) break + const where = h.enclosing ? ` (in ${h.enclosing.name})` : '' + const row = `\n ${h.line}: ${snippet(h.text)}${where} @${file.path}:${h.line}` + if (!budget.tryAdd(row)) break + out.push(row) + shown++ + } + if (shown < hits.length) { + const more = `\n … ${hits.length - shown} more in ${file.path}` + if (budget.tryAdd(more)) out.push(more) + } + } + + return { text: header + out.join(''), fileIds } +} diff --git a/core/src/trace.ts b/core/src/trace.ts new file mode 100644 index 0000000..1995340 --- /dev/null +++ b/core/src/trace.ts @@ -0,0 +1,210 @@ +// Call-chain tracing — the "who calls this / what does this call" tool. The +// build step already extracts CALL edges (CodeEdge.kind === 'calls', symbol → +// symbol), but `adjacency` indexes IMPORT edges only, so no tool has ever walked +// them. Agents were left to grep for callers and read every hit whole. This +// traverses the call edges the graph already stores: BFS out from a symbol along +// callers (inbound) or callees (outbound), ranked by containing-file centrality, +// rendered as an indented tree with exact `path:line` anchors, token-budgeted. +// Deterministic, no LLM. + +import { TokenBudget } from './budget.js' +import { computeCentrality, type Centrality } from './rank.js' +import type { ViewResult } from './skeleton.js' +import type { CodeEdge, CodeGraph, CodeSymbol } from './types.js' + +export type TraceDirection = 'callers' | 'callees' | 'both' + +export interface TraceCallsOptions { + /** Symbol to trace from (exact name; the highest-centrality match wins ties). */ + symbolName: string + /** callers = who calls it (inbound), callees = what it calls (outbound). Default callers. */ + direction?: TraceDirection + /** How many call hops to follow (default 3, max 6). */ + depth?: number + /** Token ceiling for the whole tree (default 1200). */ + budgetTokens?: number + /** Reuse a precomputed centrality (else computed on the fly). */ + centrality?: Centrality +} + +/** Symbol-id → in/out CALL edges (built on demand; the graph only indexes imports). */ +interface CallIndex { + byId: Map + adj: Map +} + +function buildCallIndex(graph: CodeGraph): CallIndex { + const byId = new Map() + for (const s of graph.symbols) byId.set(s.id, s) + const adj = new Map() + const get = (id: number) => { + let e = adj.get(id) + if (!e) { + e = { in: [], out: [] } + adj.set(id, e) + } + return e + } + for (const e of graph.edges) { + if (e.kind !== 'calls') continue + get(e.sourceId).out.push(e) + get(e.targetId).in.push(e) + } + return { byId, adj } +} + +/** `name @path:line` for a symbol, with the file's language for orientation. */ +function symbolLabel(graph: CodeGraph, sym: CodeSymbol): string { + const file = graph.filesById.get(sym.fileId) + const path = file?.path ?? '?' + return `${sym.name} @${path}:${sym.startLine}` +} + +/** + * Render one direction of the call tree from `rootId`. Children are ordered by + * containing-file centrality (then path/line) so the load-bearing chain comes + * first. A visited set breaks recursion cycles; budget stops growth cleanly. + */ +function renderTree( + graph: CodeGraph, + index: CallIndex, + rootId: number, + dir: 'in' | 'out', + depth: number, + budget: TokenBudget, + centrality: Centrality, +): { lines: string[]; fileIds: Set; edgeCount: number } { + const arrow = dir === 'in' ? '←' : '→' + const lines: string[] = [] + const fileIds = new Set() + const visited = new Set([rootId]) + let edgeCount = 0 + let truncated = false + + const walk = (id: number, level: number): void => { + if (level > depth || truncated) return + const edges = index.adj.get(id)?.[dir] ?? [] + // Resolve neighbors, rank by centrality, stable tiebreak on path:line. + const neighbors = edges + .map((e) => { + const neighborId = dir === 'in' ? e.sourceId : e.targetId + const sym = index.byId.get(neighborId) + return sym ? { sym, weight: e.weight } : null + }) + .filter((x): x is { sym: CodeSymbol; weight: number } => x != null) + .sort((a, b) => { + const sa = centrality.scoreByFile.get(a.sym.fileId) ?? 0 + const sb = centrality.scoreByFile.get(b.sym.fileId) ?? 0 + if (sb !== sa) return sb - sa + const pa = graph.filesById.get(a.sym.fileId)?.path ?? '' + const pb = graph.filesById.get(b.sym.fileId)?.path ?? '' + if (pa !== pb) return pa.localeCompare(pb) + return a.sym.startLine - b.sym.startLine + }) + + for (const { sym, weight } of neighbors) { + const indent = ' '.repeat(level) + const cycle = visited.has(sym.id) + const w = weight > 1 ? ` (×${weight})` : '' + const mark = cycle ? ' ↺ (cycle)' : '' + const row = `\n${indent}${arrow} ${symbolLabel(graph, sym)}${w}${mark}` + if (!budget.tryAdd(row)) { + truncated = true + lines.push(`\n${indent}… trace truncated (budget reached)`) + return + } + lines.push(row) + fileIds.add(sym.fileId) + edgeCount++ + if (!cycle) { + visited.add(sym.id) + walk(sym.id, level + 1) + } + } + } + + walk(rootId, 1) + return { lines, fileIds, edgeCount } +} + +/** + * Trace the call graph from a named symbol. Returns a ranked, anchored, + * token-budgeted tree — the replacement for grepping for callers. When several + * symbols share the name, the most import-central one is traced and the others + * are noted so the agent can disambiguate. + */ +export function traceCalls(graph: CodeGraph, opts: TraceCallsOptions): ViewResult { + if (!opts.symbolName) return { text: 'Provide a `symbol_name` to trace.', fileIds: [] } + const centrality = opts.centrality ?? computeCentrality(graph) + const depth = Math.max(1, Math.min(6, opts.depth ?? 3)) + const direction = opts.direction ?? 'callers' + + const matches = graph.symbols + .filter((s) => s.name === opts.symbolName) + .sort((a, b) => { + const sa = centrality.scoreByFile.get(a.fileId) ?? 0 + const sb = centrality.scoreByFile.get(b.fileId) ?? 0 + if (sb !== sa) return sb - sa + const pa = graph.filesById.get(a.fileId)?.path ?? '' + const pb = graph.filesById.get(b.fileId)?.path ?? '' + return pa.localeCompare(pb) + }) + + if (matches.length === 0) { + return { + text: + `No symbol named "${opts.symbolName}". ` + + 'Use find_symbol with a pattern to discover the exact name, or search_code for a text match.', + fileIds: [], + } + } + + const index = buildCallIndex(graph) + const root = matches[0]! + const budget = new TokenBudget(opts.budgetTokens ?? 1200) + + const header = + `# call trace: ${opts.symbolName} (${direction}, depth ${depth}) — ` + + `root ${symbolLabel(graph, root)}` + + (matches.length > 1 ? ` [+${matches.length - 1} more same-named — trace the anchor if this isn't it]` : '') + + '\n' + budget.add(header) + + const out: string[] = [] + const fileIds = new Set([root.fileId]) + let totalEdges = 0 + + const sections: Array<{ title: string; dir: 'in' | 'out' }> = + direction === 'both' + ? [ + { title: 'callers (who calls it)', dir: 'in' }, + { title: 'callees (what it calls)', dir: 'out' }, + ] + : direction === 'callees' + ? [{ title: 'callees (what it calls)', dir: 'out' }] + : [{ title: 'callers (who calls it)', dir: 'in' }] + + for (const sec of sections) { + const secHeader = `\n## ${sec.title}` + if (!budget.tryAdd(secHeader)) break + out.push(secHeader) + const tree = renderTree(graph, index, root.id, sec.dir, depth, budget, centrality) + for (const id of tree.fileIds) fileIds.add(id) + totalEdges += tree.edgeCount + if (tree.edgeCount === 0) { + const none = `\n (none — no ${sec.dir === 'in' ? 'callers' : 'callees'} in the call graph)` + if (budget.tryAdd(none)) out.push(none) + } else { + out.push(...tree.lines) + } + } + + if (totalEdges === 0) { + out.push( + '\n\nNote: call edges are heuristic (resolved by name within a file or its imports). ' + + 'Dynamic/dispatched calls may be missed — fall back to search_code for a text sweep.', + ) + } + + return { text: header + out.join(''), fileIds: [...fileIds] } +} diff --git a/core/src/types.ts b/core/src/types.ts index 635b970..3656a80 100644 --- a/core/src/types.ts +++ b/core/src/types.ts @@ -60,11 +60,29 @@ export type Language = | 'eda' | 'other' -/** Symbol kinds, deliberately aligned with the frontend SymbolKind enum. */ -export type SymbolKind = 'function' | 'class' | 'const' | 'type' | 'interface' +/** Symbol kinds, deliberately aligned with the frontend SymbolKind enum. + * `method` = a function that is a member of a class/object (nested), kept + * distinct from top-level `function` so the graph mirrors codebase-memory's + * Function/Method split and the enclosing class shows in the nodeId chain. */ +export type SymbolKind = 'function' | 'method' | 'class' | 'const' | 'type' | 'interface' + +/** + * A half-open `[start, end)` source range in **UTF-16 code units** (whole-file + * offsets). This is the unit web-tree-sitter reports and the unit VS Code's + * Position/positionAt consumes, so it converts to an editor Position with no + * encoding step — see core/src/position.ts. Never store byte offsets here. + */ +export type OffsetRange = [start: number, end: number] export interface CodeFile { id: number + /** + * Stable, root-qualified string identity (`::`) — the external + * identity for caches and editor state. Survives reindexes of unchanged files; + * see core/src/nodeid.ts. The numeric `id` is an internal, per-build adjacency + * index and must not be used as cross-reindex identity. + */ + nodeId: string /** Repo-relative POSIX path, e.g. "lib/api/client.ts". */ path: string language: Language @@ -74,10 +92,26 @@ export interface CodeFile { sha: string /** File mtime in ms (used by --watch; not part of ranking). */ lastModified: number + /** + * UTF-16 offset at which each line starts; `lineStarts[0] === 0`, one entry + * per line. Turns a symbol's UTF-16 range offset into an editor `Position` + * (line + character) via `offsetToPosition`. Recomputed each build from file + * content — held in memory, not serialized. + */ + lineStarts: number[] + /** graphRevision at which this file last changed (its sha changed). */ + revision: number } export interface CodeSymbol { id: number + /** + * Stable, root-qualified structural identity, e.g. + * `::src/auth/service.ts#AuthService.login`. Unchanged by edits to the + * body; see core/src/nodeid.ts. External identity for the UI; the numeric `id` + * is internal only. + */ + nodeId: string fileId: number name: string kind: SymbolKind @@ -86,25 +120,55 @@ export interface CodeSymbol { /** 1-based inclusive line range of the declaration. */ startLine: number endLine: number + /** + * UTF-16 range of the identifier alone — the click-to-jump target. Slicing the + * file source by this range yields exactly `name`. + */ + nameRange: OffsetRange + /** + * UTF-16 range of the whole declaration (incl. decorators/modifiers). Editor + * features anchor here (gutter annotations, scoped views). Always encloses + * `nameRange`. + */ + fullRange: OffsetRange /** True if the symbol is part of the file's public surface (exported). */ exported: boolean + /** + * graphRevision at which this symbol's rendered surface (name/kind/signature/ + * ranges/exported) last changed. A body-only edit does not advance it. + */ + revision: number } /** * A directed edge. For `import`, source/target are FILE ids (file imports file). - * For `calls`, source/target are SYMBOL ids (function calls function — heuristic, - * resolved by name within the same file or an imported file). + * For `calls`/`extends`/`implements`, source/target are SYMBOL ids — a function + * calls a function, or a class extends/implements a class or interface + * (heuristic, resolved by name within the same file or an imported file). */ export interface CodeEdge { id: number + /** + * Stable string identity derived from the endpoints' nodeIds + * (`:->`), so an edge survives reindex as long as + * both endpoints do. See core/src/nodeid.ts. + */ + nodeId: string sourceId: number targetId: number - kind: 'import' | 'calls' + kind: 'import' | 'calls' | 'extends' | 'implements' /** How many import specifiers / call sites back this edge (≥1). */ weight: number + /** graphRevision at which this edge last changed (appeared or weight changed). */ + revision: number } export interface CodeGraph { + /** + * Global monotonic build revision — bumped once per completed reindex. Returned + * on every response so consumers can detect a stale view; see core/src/revision.ts. + */ + revision: number /** Absolute repo root that was scanned. */ rootPath: string /** Display name (basename of rootPath). */ @@ -134,9 +198,23 @@ export interface RawCall { line: number } -/** Per-file parse output. */ +/** A raw inheritance reference: a supertype name written in a class/interface + * heritage clause + the line, plus whether it is an `extends` (superclass / + * interface extension) or `implements` (interface implementation). The subclass + * is the enclosing symbol at `line`; the supertype is resolved to a symbol in + * build, exactly like a call. */ +export interface RawInherit { + supertype: string + relation: 'extends' | 'implements' + /** 1-based line of the heritage clause. */ + line: number +} + +/** Per-file parse output. `nodeId` is omitted here — it is computed post-parse + * from cross-file containment/ordinals in core/src/nodeid.ts, not at parse time. */ export interface ParseResult { - symbols: Omit[] + symbols: Omit[] imports: RawImport[] calls: RawCall[] + inherits: RawInherit[] } diff --git a/core/tests/concurrency.test.ts b/core/tests/concurrency.test.ts new file mode 100644 index 0000000..f91951d --- /dev/null +++ b/core/tests/concurrency.test.ts @@ -0,0 +1,151 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { RwLock } from '../src/lock.js' +import { CancellationTokenSource, CancellationError, NONE } from '../src/cancellation.js' + +const tick = () => new Promise((r) => setTimeout(r, 0)) +function defer() { + let resolve!: () => void + const promise = new Promise((r) => (resolve = r)) + return { promise, resolve } +} + +// ---- RwLock ---------------------------------------------------------------- + +test('reads run concurrently', async () => { + const lock = new RwLock() + let current = 0 + let max = 0 + await Promise.all( + Array.from({ length: 5 }, () => + lock.read(async () => { + current++ + max = Math.max(max, current) + await tick() + current-- + }), + ), + ) + assert.ok(max >= 2, `reads should overlap (max concurrency ${max})`) +}) + +test('writes are exclusive against reads and other writes', async () => { + const lock = new RwLock() + let active = 0 + let violation = false + const guardWrite = () => + lock.write(async () => { + active++ + if (active !== 1) violation = true + await tick() + active-- + }) + const guardRead = () => + lock.read(async () => { + // While a writer holds the lock, active must be 0. + if (active !== 0) violation = true + await tick() + }) + await Promise.all([guardWrite(), guardRead(), guardWrite(), guardRead(), guardWrite()]) + assert.equal(violation, false, 'no read/write or write/write overlap') +}) + +test('two concurrent reads complete while a reindex (write) is queued behind them', async () => { + const lock = new RwLock() + const order: string[] = [] + const r1 = defer() + const r2 = defer() + + const p1 = lock.read(async () => { + order.push('r1-start') + await r1.promise + order.push('r1-end') + }) + const p2 = lock.read(async () => { + order.push('r2-start') + await r2.promise + order.push('r2-end') + }) + + await tick() + assert.deepEqual(order, ['r1-start', 'r2-start'], 'both reads started concurrently') + + const w = lock.write(async () => { + order.push('w-start') + }) + await tick() + assert.ok(!order.includes('w-start'), 'write is queued behind the active readers') + + r1.resolve() + r2.resolve() + await Promise.all([p1, p2, w]) + assert.deepEqual(order, ['r1-start', 'r2-start', 'r1-end', 'r2-end', 'w-start']) +}) + +test('a queued writer blocks later readers (no writer starvation)', async () => { + const lock = new RwLock() + const order: string[] = [] + const hold = defer() + + const firstRead = lock.read(async () => { + order.push('read1') + await hold.promise + }) + await tick() + const write = lock.write(async () => order.push('write')) + const lateRead = lock.read(async () => order.push('read2')) + await tick() + + hold.resolve() + await Promise.all([firstRead, write, lateRead]) + // The writer arrived before read2, so it must run first. + assert.deepEqual(order, ['read1', 'write', 'read2']) +}) + +test('a throwing critical section releases the lock', async () => { + const lock = new RwLock() + await assert.rejects(lock.write(async () => { throw new Error('boom') }), /boom/) + await assert.rejects(lock.read(async () => { throw new Error('bang') }), /bang/) + assert.equal(await lock.read(async () => 42), 42, 'lock still usable after errors') + assert.deepEqual(lock.state, { readers: 0, writer: false, waiting: 0 }) +}) + +// ---- Cancellation ---------------------------------------------------------- + +test('token reflects cancellation and notifies listeners once', () => { + const src = new CancellationTokenSource() + assert.equal(src.token.isCancellationRequested, false) + + let fired = 0 + src.token.onCancellationRequested(() => fired++) + src.cancel() + src.cancel() // idempotent + + assert.equal(src.token.isCancellationRequested, true) + assert.equal(fired, 1, 'listener fires exactly once') +}) + +test('throwIfCancelled throws only after cancel', () => { + const src = new CancellationTokenSource() + src.token.throwIfCancelled() // no-op + src.cancel() + assert.throws(() => src.token.throwIfCancelled(), CancellationError) +}) + +test('listener registered after cancel fires immediately; dispose unregisters', () => { + const src = new CancellationTokenSource() + const before = { hit: false } + const sub = src.token.onCancellationRequested(() => (before.hit = true)) + sub.dispose() + src.cancel() + assert.equal(before.hit, false, 'disposed listener does not fire') + + let late = false + src.token.onCancellationRequested(() => (late = true)) + assert.equal(late, true, 'listener added after cancel fires immediately') +}) + +test('NONE token is never cancelled', () => { + assert.equal(NONE.isCancellationRequested, false) + NONE.throwIfCancelled() +}) diff --git a/core/tests/fixtures/repo/sample.ts b/core/tests/fixtures/repo/sample.ts new file mode 100644 index 0000000..350ff3f --- /dev/null +++ b/core/tests/fixtures/repo/sample.ts @@ -0,0 +1,18 @@ +// Fixture repo for T1b range extraction. The emoji + CJK line sits ABOVE `beta` +// and `Widget`, so a byte-based offset would drift their ranges. UTF-16 offsets +// must still slice to the exact identifier. +export function alpha() { + return 1 +} + +const flag = "🚩 banner 中文 café" + +export function beta(x: number): number { + return alpha() + x +} + +export class Widget { + render(): string { + return 'x' + } +} diff --git a/core/tests/fixtures/utf16-sample.ts b/core/tests/fixtures/utf16-sample.ts new file mode 100644 index 0000000..86737a2 --- /dev/null +++ b/core/tests/fixtures/utf16-sample.ts @@ -0,0 +1,29 @@ +// UTF-16 conversion fixture. Deliberately mixes an astral emoji (surrogate pair, +// 2 UTF-16 units), CJK (1 unit each), and a combining mark (base + U+0301, 2 +// units) ABOVE and WITHIN the lines we assert on — the exact conditions under +// which a byte-based offset silently drifts the column. Positions below are +// hand-verified UTF-16 code-unit offsets. Reused by the T1b range tests. + +// L0: const a = 1 +// L1: const 🚩 = "x" 🚩 = U+1F6A9 (surrogate pair, 2 units) +// L2: let 中文 = 42 +// L3: const café = 7 é written as 'e' + U+0301 combining acute (2 units) +// L4: end +export const source = 'const a = 1\nconst 🚩 = "x"\nlet 中文 = 42\nconst café = 7\nend' + +/** Expected UTF-16 offset at which each line begins. */ +export const expectedLineStarts = [0, 12, 27, 39, 55] + +/** offset (whole-file, UTF-16) → expected 0-based Position. Each is a regression + * guard: a byte-based implementation produces different `character` values. */ +export const cases: { offset: number; line: number; character: number; note: string }[] = [ + { offset: 0, line: 0, character: 0, note: 'start of file' }, + { offset: 6, line: 0, character: 6, note: 'ascii identifier `a` line' }, + { offset: 18, line: 1, character: 6, note: 'start of 🚩 (surrogate pair)' }, + { offset: 20, line: 1, character: 8, note: 'space AFTER 🚩 — proves emoji counts as 2 units' }, + { offset: 31, line: 2, character: 4, note: 'CJK 中' }, + { offset: 34, line: 2, character: 7, note: '`=` on the CJK line' }, + { offset: 49, line: 3, character: 10, note: 'combining acute after café' }, + { offset: 51, line: 3, character: 12, note: '`=` after café (combining mark counted)' }, + { offset: 55, line: 4, character: 0, note: 'start of last line `end`' }, +] diff --git a/core/tests/multiroot.test.ts b/core/tests/multiroot.test.ts new file mode 100644 index 0000000..345ecbc --- /dev/null +++ b/core/tests/multiroot.test.ts @@ -0,0 +1,95 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { RootRegistry } from '../src/roots.js' + +function tmpRepo(tag: string, files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `ov-root-${tag}-`)) + for (const [name, content] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, name), content) + } + return dir +} + +test('two roots register, build, and query independently with no id/edge bleed', async () => { + const dirA = tmpRepo('a', { 'a.ts': 'export function fromA(): number { return 1 }\n' }) + const dirB = tmpRepo('b', { 'b.ts': 'export function fromB(): number { return 2 }\n' }) + try { + const reg = new RootRegistry({ buildOptions: { parseTimeoutMs: 0 } }) + const idA = reg.register(dirA) + const idB = reg.register(dirB) + + assert.notEqual(idA, idB, 'distinct roots → distinct rootIds') + assert.equal(reg.list().length, 2) + assert.equal(reg.register(dirA), idA, 'register is idempotent') + assert.equal(reg.list().length, 2, 're-registering does not duplicate') + + const gA = await reg.build(idA) + const gB = await reg.build(idB) + + // Content isolation: each graph sees only its own symbols. + assert.ok(gA.symbols.some((s) => s.name === 'fromA')) + assert.ok(!gA.symbols.some((s) => s.name === 'fromB')) + assert.ok(gB.symbols.some((s) => s.name === 'fromB')) + assert.ok(!gB.symbols.some((s) => s.name === 'fromA')) + + // No id bleed: every node of A is namespaced by idA, every node of B by idB, + // and the two id sets are disjoint. + // Every node id carries its own rootId (edges embed both endpoints' ids), and + // the two id sets are disjoint. + const idsA = [...gA.files, ...gA.symbols, ...gA.edges].map((n) => n.nodeId) + const idsB = [...gB.files, ...gB.symbols, ...gB.edges].map((n) => n.nodeId) + assert.ok(idsA.every((id) => id.includes(idA)), 'A ids namespaced by idA') + assert.ok(idsB.every((id) => id.includes(idB)), 'B ids namespaced by idB') + assert.equal(idsA.filter((id) => idsB.includes(id)).length, 0, 'no shared nodeIds') + + reg.closeAll() + } finally { + fs.rmSync(dirA, { recursive: true, force: true }) + fs.rmSync(dirB, { recursive: true, force: true }) + } +}) + +test('unregister removes a root; rootIdForPath is stable and matches register', async () => { + const dir = tmpRepo('c', { 'c.ts': 'export const v = 1\n' }) + try { + const reg = new RootRegistry({ buildOptions: { parseTimeoutMs: 0 } }) + const precomputed = reg.rootIdForPath(dir) + const id = reg.register(dir) + assert.equal(id, precomputed, 'rootIdForPath predicts the registered id') + assert.ok(reg.has(id)) + + reg.unregister(id) + assert.ok(!reg.has(id)) + assert.equal(reg.list().length, 0) + reg.unregister(id) // unregistering an unknown root is a no-op, must not throw + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +test('reindexing one root does not disturb another', async () => { + const dirA = tmpRepo('ra', { 'a.ts': 'export function a1(): number { return 1 }\n' }) + const dirB = tmpRepo('rb', { 'b.ts': 'export function b1(): number { return 1 }\n' }) + try { + const reg = new RootRegistry({ buildOptions: { parseTimeoutMs: 0 } }) + const idA = reg.register(dirA) + const idB = reg.register(dirB) + await reg.build(idA) + await reg.build(idB) + + // Change A, reindex A; B's revision counter is independent. + fs.appendFileSync(path.join(dirA, 'a.ts'), 'export function a2(): number { return 2 }\n') + const rA = await reg.reindex(idA) + const rB = await reg.reindex(idB) + + assert.ok(rA.changes.changedNodeIds.some((id) => id.includes('a2')), 'A saw its new symbol') + assert.ok(!rB.changes.changedNodeIds.some((id) => id.includes('a2')), 'B unaffected by A edit') + reg.closeAll() + } finally { + fs.rmSync(dirA, { recursive: true, force: true }) + fs.rmSync(dirB, { recursive: true, force: true }) + } +}) diff --git a/core/tests/nodeid.test.ts b/core/tests/nodeid.test.ts new file mode 100644 index 0000000..e4914e4 --- /dev/null +++ b/core/tests/nodeid.test.ts @@ -0,0 +1,98 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import * as path from 'node:path' +import { fileURLToPath } from 'node:url' +import { computeSymbolNodeIds, rootIdFor, fileNodeId } from '../src/nodeid.js' +import { buildGraph } from '../src/build.js' +import type { CodeGraph } from '../src/types.js' + +const R = 'root0' +const F = 'src/f.ts' +const sym = (name: string, s: number, e: number) => ({ name, fullRange: [s, e] as [number, number] }) + +// ---- pure unit tests: containment + ordinals ------------------------------- + +test('top-level symbols with distinct names get suffix-free structural ids', () => { + const ids = computeSymbolNodeIds(R, F, [sym('alpha', 0, 10), sym('beta', 20, 30)]) + assert.deepEqual(ids, [`${R}::${F}#alpha`, `${R}::${F}#beta`]) +}) + +test('an enclosed symbol is qualified by its container chain', () => { + // Outer [0,100) contains method [20,40); Mid [10,90) sits between for depth. + const ids = computeSymbolNodeIds(R, F, [ + sym('Outer', 0, 100), + sym('Mid', 10, 90), + sym('leaf', 20, 40), + ]) + assert.deepEqual(ids, [ + `${R}::${F}#Outer`, + `${R}::${F}#Outer.Mid`, + `${R}::${F}#Outer.Mid.leaf`, + ]) +}) + +test('identical structural paths get stable source-order ordinals', () => { + const ids = computeSymbolNodeIds(R, F, [sym('twin', 0, 10), sym('twin', 20, 30)]) + assert.deepEqual(ids, [`${R}::${F}#twin@0`, `${R}::${F}#twin@1`]) +}) + +test('same method name in two different classes does NOT collide (no ordinal)', () => { + const ids = computeSymbolNodeIds(R, F, [ + sym('A', 0, 50), + sym('run', 10, 20), // A.run + sym('B', 60, 110), + sym('run', 70, 80), // B.run + ]) + assert.deepEqual(ids, [ + `${R}::${F}#A`, + `${R}::${F}#A.run`, + `${R}::${F}#B`, + `${R}::${F}#B.run`, + ]) +}) + +test('rootIdFor is deterministic, path-sensitive, and 12 hex chars', () => { + const a = rootIdFor('/Users/me/proj') + assert.equal(a, rootIdFor('/Users/me/proj')) + assert.notEqual(a, rootIdFor('/Users/me/other')) + assert.match(a, /^[0-9a-f]{12}$/) +}) + +test('fileNodeId format', () => { + assert.equal(fileNodeId('abc', 'a/b.ts'), 'abc::a/b.ts') +}) + +// ---- integration: through the real parser + build -------------------------- + +const here = path.dirname(fileURLToPath(import.meta.url)) +const repoDir = path.join(here, 'fixtures', 'repo') + +test('build assigns well-formed, unique nodeIds to files/symbols/edges', async () => { + const graph: CodeGraph = await buildGraph(repoDir, { parseTimeoutMs: 0 }) + const rootId = rootIdFor(path.resolve(repoDir)) + + for (const f of graph.files) { + assert.equal(f.nodeId, `${rootId}::${f.path}`) + } + // Widget.render must be qualified by its enclosing class. + const names = graph.symbols.map((s) => s.nodeId) + assert.ok( + names.some((id) => id.endsWith('#Widget.render')), + `expected a #Widget.render id, got: ${names.join(', ')}`, + ) + assert.ok(names.some((id) => id.endsWith('#alpha'))) + assert.ok(names.some((id) => id.endsWith('#beta'))) + + // Uniqueness across all nodes. + const all = [...graph.files, ...graph.symbols, ...graph.edges].map((n) => n.nodeId) + assert.equal(new Set(all).size, all.length, 'all nodeIds unique') + assert.ok(graph.edges.every((e) => e.nodeId.length > 0), 'edges have nodeIds') +}) + +test('nodeIds are byte-identical across two reindexes of an unchanged repo', async () => { + const a = await buildGraph(repoDir, { parseTimeoutMs: 0 }) + const b = await buildGraph(repoDir, { parseTimeoutMs: 0 }) + const ids = (g: CodeGraph) => + [...g.files, ...g.symbols, ...g.edges].map((n) => n.nodeId).sort() + assert.deepEqual(ids(a), ids(b)) +}) diff --git a/core/tests/position.test.ts b/core/tests/position.test.ts new file mode 100644 index 0000000..41ceb07 --- /dev/null +++ b/core/tests/position.test.ts @@ -0,0 +1,77 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { offsetToPosition, computeLineStarts } from '../src/position.js' +import { source, expectedLineStarts, cases } from './fixtures/utf16-sample.js' + +// Independent reference: a straight linear scan in UTF-16 code units. Correct by +// construction (it's the definition VS Code's positionAt implements); used to +// fuzz-check the binary-search helper across every offset. +function refPosition(src: string, offset: number): { line: number; character: number } { + const off = Math.max(0, Math.min(offset, src.length)) + let line = 0 + let lineStart = 0 + for (let i = 0; i < off; i++) { + if (src.charCodeAt(i) === 10) { + line++ + lineStart = i + 1 + } + } + return { line, character: off - lineStart } +} + +test('computeLineStarts matches the hand-verified UTF-16 line offsets', () => { + assert.deepEqual(computeLineStarts(source), expectedLineStarts) +}) + +test('hand-verified multibyte cases (emoji / CJK / combining) resolve exactly', () => { + const lineStarts = computeLineStarts(source) + for (const c of cases) { + assert.deepEqual( + offsetToPosition(lineStarts, c.offset), + { line: c.line, character: c.character }, + `offset ${c.offset} (${c.note})`, + ) + } +}) + +test('fixture case offsets actually land where the notes claim (guards the fixture itself)', () => { + // Every line start in `cases` with character 0 must sit at a real line boundary. + for (const c of cases) { + if (c.character === 0 && c.offset > 0) { + assert.equal(source.charCodeAt(c.offset - 1), 10, `offset ${c.offset} should follow a \\n`) + } + } +}) + +test('agrees with a linear UTF-16 reference for every in-contract offset', () => { + // Contract domain is [0, source.length] — tree-sitter node offsets never exceed + // the source length. Negatives clamp to start; past-EOF is out of contract and + // covered explicitly below. Within the domain the two implementations must agree. + const lineStarts = computeLineStarts(source) + for (let off = -2; off <= source.length; off++) { + assert.deepEqual( + offsetToPosition(lineStarts, off), + refPosition(source, off), + `offset ${off}`, + ) + } +}) + +test('each line start resolves to character 0 on its own line', () => { + const lineStarts = computeLineStarts(source) + lineStarts.forEach((start, line) => { + assert.deepEqual(offsetToPosition(lineStarts, start), { line, character: 0 }) + }) +}) + +test('edge cases: negative, past-EOF, single-line, empty lineStarts', () => { + const lineStarts = computeLineStarts(source) + assert.deepEqual(offsetToPosition(lineStarts, -5), { line: 0, character: 0 }) + const last = lineStarts.length - 1 + assert.deepEqual(offsetToPosition(lineStarts, source.length + 100), { + line: last, + character: source.length + 100 - expectedLineStarts[last]!, + }) + assert.deepEqual(offsetToPosition([0], 4), { line: 0, character: 4 }) + assert.deepEqual(offsetToPosition([], 7), { line: 0, character: 7 }) +}) diff --git a/core/tests/ranges.test.ts b/core/tests/ranges.test.ts new file mode 100644 index 0000000..dc5715e --- /dev/null +++ b/core/tests/ranges.test.ts @@ -0,0 +1,75 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import * as fs from 'node:fs' +import * as path from 'node:path' +import { fileURLToPath } from 'node:url' +import { buildGraph } from '../src/build.js' +import { offsetToPosition } from '../src/position.js' +import type { CodeGraph } from '../src/types.js' + +const here = path.dirname(fileURLToPath(import.meta.url)) +const repoDir = path.join(here, 'fixtures', 'repo') + +// Build the fixture graph once — grammar loading is the slow part. +// parseTimeoutMs: 0 disables the per-file timeout: the fixture parses instantly, +// and it sidesteps a pre-existing leaked-setTimeout in parseSource that otherwise +// keeps the test process alive ~10s after the last assertion. +let cached: CodeGraph | null = null +async function graphOnce(): Promise { + if (!cached) cached = await buildGraph(repoDir, { parseTimeoutMs: 0 }) + return cached +} + +function sourceOf(graph: CodeGraph, fileId: number): string { + const f = graph.filesById.get(fileId)! + return fs.readFileSync(path.join(repoDir, f.path), 'utf8') +} + +test('T1b: 100% of symbols carry a nameRange that slices to exactly the name', async () => { + const graph = await graphOnce() + assert.ok(graph.symbols.length > 0, 'fixture should yield symbols') + for (const s of graph.symbols) { + assert.equal(s.nameRange.length, 2, `${s.name}: nameRange is a 2-tuple`) + assert.equal(s.fullRange.length, 2, `${s.name}: fullRange is a 2-tuple`) + const src = sourceOf(graph, s.fileId) + assert.equal( + src.slice(s.nameRange[0], s.nameRange[1]), + s.name, + `${s.name}: nameRange must slice to the identifier`, + ) + assert.ok( + s.fullRange[0] <= s.nameRange[0] && s.nameRange[1] <= s.fullRange[1], + `${s.name}: fullRange must enclose nameRange`, + ) + } +}) + +test('T1b: every file has UTF-16 lineStarts beginning at 0', async () => { + const graph = await graphOnce() + for (const f of graph.files) { + assert.ok(f.lineStarts.length >= 1, `${f.path}: has lineStarts`) + assert.equal(f.lineStarts[0], 0, `${f.path}: lineStarts[0] === 0`) + } +}) + +test('T1b: fullRange offsets convert back to the declared start/end lines', async () => { + const graph = await graphOnce() + for (const s of graph.symbols) { + const f = graph.filesById.get(s.fileId)! + const start = offsetToPosition(f.lineStarts, s.fullRange[0]) + const end = offsetToPosition(f.lineStarts, s.fullRange[1]) + assert.equal(start.line + 1, s.startLine, `${s.name}: fullRange start → startLine`) + assert.equal(end.line + 1, s.endLine, `${s.name}: fullRange end → endLine`) + } +}) + +test('T1b: multibyte content above a symbol does not drift its range', async () => { + const graph = await graphOnce() + const beta = graph.symbols.find((s) => s.name === 'beta') + assert.ok(beta, '`beta` should be extracted') + const src = sourceOf(graph, beta!.fileId) + // `beta` is declared after a line with an emoji (surrogate pair), CJK, and an + // accented char. A byte-based offset would land mid-string; UTF-16 slices clean. + assert.equal(src.slice(beta!.nameRange[0], beta!.nameRange[1]), 'beta') + assert.ok(beta!.startLine > 7, '`beta` is below the multibyte line') +}) diff --git a/core/tests/revision.test.ts b/core/tests/revision.test.ts new file mode 100644 index 0000000..546856a --- /dev/null +++ b/core/tests/revision.test.ts @@ -0,0 +1,86 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Indexer } from '../src/build.js' +import type { CodeGraph } from '../src/types.js' + +function tmpRepo(files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-rev-')) + for (const [name, content] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, name), content) + } + return dir +} + +const nodeById = (g: CodeGraph) => { + const m = new Map() + for (const n of [...g.files, ...g.symbols, ...g.edges]) m.set(n.nodeId, n.revision) + return m +} +const fileRev = (g: CodeGraph, p: string) => g.files.find((f) => f.path === p)!.revision +const symRev = (g: CodeGraph, name: string) => g.symbols.find((s) => s.name === name)!.revision + +test('graphRevision increments by one per reindex; only changed nodes advance', async () => { + const dir = tmpRepo({ + 'a.ts': 'export function alpha(): number { return 1 }\nexport function beta(): number { return 2 }\n', + 'b.ts': 'export function solo(): number { return 3 }\n', + }) + try { + const idx = new Indexer(dir, { parseTimeoutMs: 0 }) + + // ---- build 1 ----------------------------------------------------------- + const r1 = await idx.reindex() + assert.equal(r1.graph.revision, 1, 'first reindex → graphRevision 1') + for (const rev of nodeById(r1.graph).values()) assert.equal(rev, 1, 'all nodes at rev 1') + + // ---- edit: append a new function to a.ts (does not move existing symbols) + fs.appendFileSync(path.join(dir, 'a.ts'), 'export function gamma(): number { return 42 }\n') + const r2 = await idx.reindex() + + assert.equal(r2.graph.revision, 2, 'second reindex → graphRevision 2') + assert.equal(symRev(r2.graph, 'gamma'), 2, 'new symbol advanced') + assert.equal(fileRev(r2.graph, 'a.ts'), 2, 'edited file advanced (sha changed)') + assert.equal(symRev(r2.graph, 'alpha'), 1, 'untouched symbol stayed') + assert.equal(symRev(r2.graph, 'beta'), 1, 'untouched symbol stayed') + assert.equal(symRev(r2.graph, 'solo'), 1, 'symbol in unchanged file stayed') + assert.equal(fileRev(r2.graph, 'b.ts'), 1, 'unchanged file stayed') + + // changedNodeIds is EXACTLY the set of nodes now at the new revision. + const advanced = [...nodeById(r2.graph).entries()].filter(([, r]) => r === 2).map(([id]) => id) + assert.deepEqual([...r2.changes.changedNodeIds].sort(), advanced.sort()) + assert.ok(r2.changes.changedNodeIds.includes(r2.graph.symbols.find((s) => s.name === 'gamma')!.nodeId)) + + // ---- no-op reindex: counter still bumps, nothing changes --------------- + const r3 = await idx.reindex() + assert.equal(r3.graph.revision, 3, 'no-op reindex still bumps graphRevision') + assert.deepEqual(r3.changes.changedNodeIds, [], 'no nodes changed') + for (const rev of nodeById(r3.graph).values()) assert.notEqual(rev, 3, 'no node at rev 3') + + idx.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +test('a body-only edit advances the file but not the symbol', async () => { + const dir = tmpRepo({ + 'x.ts': 'export function keep(): number {\n return 1\n}\n', + }) + try { + const idx = new Indexer(dir, { parseTimeoutMs: 0 }) + const r1 = await idx.reindex() + const before = symRev(r1.graph, 'keep') + + // Change the body only — same signature, same line count, same ranges. + fs.writeFileSync(path.join(dir, 'x.ts'), 'export function keep(): number {\n return 2\n}\n') + const r2 = await idx.reindex() + + assert.equal(fileRev(r2.graph, 'x.ts'), 2, 'file advanced (content changed)') + assert.equal(symRev(r2.graph, 'keep'), before, 'symbol surface unchanged → revision held') + idx.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) diff --git a/docs/GRAPH.md b/docs/GRAPH.md index 12b886f..50ae175 100644 --- a/docs/GRAPH.md +++ b/docs/GRAPH.md @@ -190,9 +190,11 @@ additive, and bound to `127.0.0.1` only. `mcp/hooks/` ships two optional Claude Code hooks — wire them into a repo's `.claude/settings.json`: -- **`openvisio-gate.mjs`** (`PreToolUse`) — blocks `Read`/`Grep`/`Glob`/`Bash` - until an `openvisio` tool has primed the session, so the agent consults the - graph *first*. After priming, everything passes. +- **`openvisio-gate.mjs`** (`PreToolUse`) — gates code *search* (`Grep`/`Glob` + and grep/find-style `Bash`) until an `openvisio` tool has primed the session, + steering discovery to `search_code`/`find_symbol`/`trace_calls` first. `Read` + is never blocked (the goal is fewer blind whole-file reads, not no reading), + and non-code Grep/Glob always pass. After priming, everything passes. - **`openvisio-instruct.mjs`** (`UserPromptSubmit`) — pulls a pending instruction from the spotlight server and injects it as context on the next prompt, so a question typed in the viewer reaches the agent with no copy-paste. Fails open diff --git a/mcp/README.md b/mcp/README.md index 42b182b..5cd1d4f 100644 --- a/mcp/README.md +++ b/mcp/README.md @@ -104,9 +104,11 @@ agent reads only the slice it needs. |------|--------------| | `resolve_context` | task description → task-ranked skeleton + the neighborhoods of the most relevant files. **Call this first.** | | `get_repo_skeleton` | the whole ranked repo map: most import-central files + their public symbols | -| `find_symbol` | locate a function/class/type by name or pattern → signature + anchor | +| `find_symbol` | locate a function/class/type by name, regex, or **natural-language query** (BM25 — "update cloud client" → `updateCloudClient`) → signature + anchor | +| `search_code` | full-text search over the indexed repo — **the grep/ripgrep replacement**. Any literal, regex, TODO, or error string → ranked, symbol-annotated, anchored hits | +| `trace_calls` | walk the **call graph** — who calls a function (impact) or what it calls, multi-hop, anchored tree. Replaces grepping for call sites | | `get_neighborhood` | local import subgraph around a file/symbol (dependents + dependencies) | -| `get_dependents` | who imports this — directed impact analysis | +| `get_dependents` | who imports this — directed file-level impact analysis | | `get_hotspots` | load-bearing / risky files: high import centrality (+ git churn) | The tool surface is intentionally tiny — schemas load into the agent's context diff --git a/mcp/hooks/openvisio-gate.mjs b/mcp/hooks/openvisio-gate.mjs index c0710a1..64f5da9 100755 --- a/mcp/hooks/openvisio-gate.mjs +++ b/mcp/hooks/openvisio-gate.mjs @@ -9,13 +9,17 @@ // again on the next task. // • Write commands also delete .openvisio/graph.json so the viewer // re-indexes on the next request. -// • Read/Grep/Glob for non-code files (configs, docs, lockfiles) are always -// allowed without priming. -// • All other code-discovery tools (Read/Grep/Glob on code, grep/find/etc. -// in Bash) are DENIED until an openvisio tool primes the session. +// • Read is ALWAYS allowed — the goal is to reduce blind whole-file reads, not +// to block reading. Agents still read the anchored slices the tools hand +// them; we just don't gate Read itself. +// • Grep/Glob for non-code files (configs, docs, lockfiles) are always allowed +// without priming. +// • The SEARCH tools (Grep/Glob on code, grep/find/etc. in Bash) are DENIED +// until an openvisio tool primes the session — steering discovery to +// search_code / find_symbol / trace_calls instead of blind text sweeps. // // The first openvisio tool call in each task "primes" the session; after that -// all tools pass (anchored reads, git status, etc. all flow freely). +// all tools pass (search, git status, etc. all flow freely). // // Install in the repo you point the agent at — .claude/settings.json: // { @@ -159,9 +163,10 @@ if (toolName === 'Bash') { hookEventName: 'PreToolUse', permissionDecision: 'deny', permissionDecisionReason: - 'openvisio-gate: call the openvisio MCP first. Before searching code, ' + - 'call `resolve_context` with your task (then find_symbol / get_neighborhood ' + - '/ get_dependents) to get path:line anchors. Then retry this action.', + 'openvisio-gate: use the openvisio MCP instead of grep/rg/find. For text ' + + 'search call `search_code` (the grep replacement — ranked, anchored, ' + + 'symbol-annotated hits). To orient first, call `resolve_context`, then ' + + 'find_symbol / get_neighborhood / get_dependents for path:line anchors.', }, }), ) @@ -177,20 +182,22 @@ if (toolName === 'Bash') { if (existsSync(marker)) process.exit(0) // --------------------------------------------------------------------------- -// Rule 4: Ungated tools (non-Bash, non-Read/Grep/Glob) pass through. +// Rule 4: Read always passes, and any other ungated tool (non-Bash, non-Grep/ +// Glob) passes through. Reading is never blocked — we only steer SEARCH. // --------------------------------------------------------------------------- -const GATED_TOOLS = new Set(['Read', 'Grep', 'Glob']) +const GATED_TOOLS = new Set(['Grep', 'Glob']) if (!GATED_TOOLS.has(toolName)) process.exit(0) // --------------------------------------------------------------------------- -// Rule 5: Non-code file reads are always allowed. +// Rule 5: Grep/Glob over non-code files (configs, docs, lockfiles) are allowed. // --------------------------------------------------------------------------- if (isNonCodeRead(toolName, toolInput)) process.exit(0) // --------------------------------------------------------------------------- -// Rule 6: Deny — call openvisio first. +// Rule 6: Deny the SEARCH (Grep/Glob over code) — call openvisio first. +// (Read is never gated; this only catches blind text/glob sweeps.) // --------------------------------------------------------------------------- process.stdout.write( @@ -199,10 +206,10 @@ process.stdout.write( hookEventName: 'PreToolUse', permissionDecision: 'deny', permissionDecisionReason: - 'openvisio-gate: call the openvisio MCP first. Before reading, grepping, or ' + - 'globbing any file in this repo, call `resolve_context` with your task (then ' + - 'find_symbol / get_neighborhood / get_dependents) to get path:line anchors. ' + - 'Then retry this action.', + 'openvisio-gate: use the openvisio MCP for code search instead of Grep/Glob. ' + + 'For text search call `search_code` (the grep replacement); for a symbol by ' + + 'name/concept use `find_symbol`; for callers use `trace_calls`. To orient, call ' + + '`resolve_context` first. Reading files is always allowed. Then retry this action.', }, }), ) diff --git a/mcp/smoke.mjs b/mcp/smoke.mjs index 65adf15..2900bbf 100644 --- a/mcp/smoke.mjs +++ b/mcp/smoke.mjs @@ -1,5 +1,5 @@ // End-to-end smoke test for the OpenVisio MCP server. Spins up the built server -// over stdio against a disposable repo and exercises every phase: the 7 tools, +// over stdio against a disposable repo and exercises every phase: the 9 tools, // the spotlight SSE stream, incremental --watch, and determinism. Run with: // // npm run smoke (builds first) — or node mcp/smoke.mjs @@ -35,7 +35,7 @@ fs.rmSync(root, { recursive: true, force: true }) fs.mkdirSync(path.join(root, 'src'), { recursive: true }) fs.writeFileSync(path.join(root, 'src/types.ts'), `export interface User { id: number }\nexport type Role = 'admin' | 'user'\n`) fs.writeFileSync(path.join(root, 'src/auth.ts'), `import { User } from './types'\nexport function login(u: User){ return u.id }\n`) -fs.writeFileSync(path.join(root, 'src/api.ts'), `import { login } from './auth'\nimport { Role } from './types'\nexport function handler(){ return login }\n`) +fs.writeFileSync(path.join(root, 'src/api.ts'), `import { login } from './auth'\nimport { Role } from './types'\nexport function handler(){ return login({ id: 1 }) }\n`) const transport = new StdioClientTransport({ command: 'node', @@ -50,12 +50,18 @@ const call = async (name, args = {}) => (await client.callTool({ name, arguments console.log('\n[tools] surface + output') const tools = (await client.listTools()).tools.map((t) => t.name) -// 7 read-only tools + the reverse channel (get_user_request + submit_answer), present under --spotlight. -ok(tools.length === 9, `lists 9 tools: ${tools.join(', ')}`) +// 9 read-only tools + the reverse channel (get_user_request + submit_answer), present under --spotlight. +ok(tools.length === 11, `lists 11 tools: ${tools.join(', ')}`) ok(tools.includes('get_user_request') && tools.includes('submit_answer'), 'reverse-channel tools registered under --spotlight') ok((await call('resolve_context', { task_description: 'add admin role check to login' })).includes('auth.ts'), 'resolve_context surfaces auth.ts') ok((await call('get_repo_skeleton', { budget_tokens: 400 })).includes('@src/'), 'get_repo_skeleton has path:line anchors') ok((await call('find_symbol', { name: 'login' })).includes('src/auth.ts:'), 'find_symbol(login) returns an anchor') +ok((await call('search_code', { query: 'login' })).includes('@src/auth.ts:2'), 'search_code(login) returns an anchored content hit') +ok((await call('search_code', { query: 'admin|user', regex: true, path_filter: '*.ts' })).includes('@src/types.ts:'), 'search_code regex + glob path_filter works') +ok((await call('search_code', { query: 'nope_absent_xyz' })).includes('No matches'), 'search_code reports no matches cleanly') +ok((await call('find_symbol', { query: 'handle user login' })).includes('src/auth.ts:'), 'find_symbol(query) BM25-ranks login by natural language') +ok((await call('trace_calls', { symbol_name: 'login' })).includes('handler'), 'trace_calls(login) finds caller handler over call edges') +ok((await call('trace_calls', { symbol_name: 'handler', direction: 'callees' })).includes('login'), 'trace_calls(handler, callees) shows it calls login') ok((await call('get_dependents', { target: 'types.ts' })).includes('is imported by'), 'get_dependents(types.ts) lists importers') ok((await call('get_neighborhood', { target: 'auth.ts' })).includes('CENTER'), 'get_neighborhood(auth.ts) has a CENTER') ok((await call('get_hotspots', {})).includes('centrality'), 'get_hotspots ranks by centrality') diff --git a/mcp/src/adapter.ts b/mcp/src/adapter.ts index 6c90dfad5c31c558ddcc967ffd1aab3b8fa8ccbb..fe57530f9f62165432069b2e0ef82c9aef9d0851 100644 GIT binary patch delta 368 zcmaE3JJ(@@H;a@)je>e=MM-L2O0hbKotayZlbV~FS5my$i$#;GUSD4!0VtMVRFa^O znv$MctdN|amzSDcqL7xElUi)8kdU01lT)0ap8z!l$U>Nr09RdHnVXcKQ>@3O01_%L zQ3#Iob@KO#cXe@hjraESbFoqgN=?o$O0iJ~8whqxYDGOZm#OP0fE7X2U{zwLV5?9K zv { 'returns a task-ranked skeleton plus the neighborhoods of the most relevant ' + 'files, every line carrying a `path:line` anchor.\n' + 'Step 2 — drill in with the right tool instead of opening files:\n' + - ' • find a function/class/type by name or pattern → `find_symbol`\n' + - ' • who imports / what this imports (impact) → `get_dependents`\n' + + ' • find a symbol by name, regex, OR natural-language description → `find_symbol`\n' + + ' • any literal string / regex / TODO / error message / config key → `search_code`\n' + + ' • who CALLS a function / what it calls (call chain, impact) → `trace_calls`\n' + + ' • who imports / what this imports (file-level impact) → `get_dependents`\n' + ' • local import subgraph around a file/symbol → `get_neighborhood`\n' + ' • churn × centrality refactor/risk candidates → `get_hotspots`\n' + ' • the whole ranked repo map → `get_repo_skeleton`\n' + '\n' + 'Only after these tools have given you the `path:line` anchors may you read ' + 'source — and then read just the anchored slice, never the whole file. ' + - 'Do NOT reach for grep/glob/find/cat to discover code structure; that wastes ' + - 'tokens and bypasses the graph. The ONLY exceptions are non-code assets the ' + - 'graph does not model: config, docs/markdown, lockfiles, and binary/data files. ' + - 'When in doubt, call `resolve_context` first.', + 'NEVER reach for grep/rg/ripgrep/glob/find/cat to discover code — `search_code` ' + + 'is the grep replacement: it searches the same indexed files and returns ranked, ' + + 'symbol-annotated, anchored hits for a fraction of the tokens. The ONLY exceptions ' + + 'are non-code assets the graph does not model: config, docs/markdown, lockfiles, ' + + 'and binary/data files. When in doubt, call `resolve_context` first.', }, ) diff --git a/mcp/src/tools.ts b/mcp/src/tools.ts index 6414669..9721aa8 100644 --- a/mcp/src/tools.ts +++ b/mcp/src/tools.ts @@ -17,12 +17,17 @@ import { dependenciesOf, dependentsOf, findSymbols, + rankSymbols, resolveContext, resolveFileTarget, + searchContent, sliceSymbolSource, TokenBudget, + traceCalls, type Centrality, type CodeGraph, + type RankedSymbolHit, + type SymbolHit, } from '@openvisio/core' import type { UserRequest } from './spotlight.js' @@ -116,20 +121,27 @@ function findSymbolTool(getState: GetState): ToolDef { return { name: 'find_symbol', description: - 'Locate a function/class/type by name or pattern: returns signature, exact path:line anchor, and the (elided) definition body — no whole-file reads.', + 'Locate a function/class/type by exact name, regex pattern, OR natural-language query (BM25 — ranks by relevance, splits camelCase, so "update cloud client" finds updateCloudClient without knowing the exact name). Returns signature, exact path:line anchor, and the (elided) definition body — no whole-file reads. Use `query` when you know WHAT the code does but not its name.', inputShape: { name: z.string().optional().describe('Exact symbol name.'), pattern: z.string().optional().describe('Case-insensitive regex over symbol names.'), + query: z + .string() + .optional() + .describe('Natural-language description — BM25-ranked discovery when you do not know the exact name.'), budget_tokens: budgetArg(800), }, handler: (args) => { const { graph, centrality } = getState() const name = args.name as string | undefined const pattern = args.pattern as string | undefined - if (!name && !pattern) return { text: 'Provide `name` or `pattern`.', touchedFiles: [] } + const query = args.query as string | undefined + if (!name && !pattern && !query) return { text: 'Provide `name`, `pattern`, or `query`.', touchedFiles: [] } const budget = new TokenBudget((args.budget_tokens as number | undefined) ?? 800) - const hits = findSymbols(graph, { name, pattern, centrality }) - if (hits.length === 0) return { text: `No symbols match ${name ?? pattern}.`, touchedFiles: [] } + const hits: (SymbolHit | RankedSymbolHit)[] = query + ? rankSymbols(graph, query, { centrality }) + : findSymbols(graph, { name, pattern, centrality }) + if (hits.length === 0) return { text: `No symbols match ${name ?? pattern ?? query}.`, touchedFiles: [] } const blocks: string[] = [] const touched = new Set() @@ -152,6 +164,76 @@ function findSymbolTool(getState: GetState): ToolDef { } } +// --------------------------------------------------------------------------- +// search_code — the grep replacement. Content search over the indexed file set +// with ranked, anchored, symbol-annotated hits. Use INSTEAD of grep/rg/find. +// --------------------------------------------------------------------------- +function searchCodeTool(getState: GetState): ToolDef { + return { + name: 'search_code', + description: + 'Full-text search over the indexed repo — the grep/ripgrep replacement. Find any literal string, regex, TODO, error message, or config key. Returns matches ranked by file importance, each with its enclosing symbol and an exact path:line anchor. Use this instead of grep/rg/find/Grep.', + inputShape: { + query: z.string().describe('Literal text to find (or a regex when regex=true).'), + regex: z.boolean().optional().describe('Treat query as a JS regular expression (default false).'), + case_sensitive: z.boolean().optional().describe('Case-sensitive match (default false).'), + path_filter: z + .string() + .optional() + .describe('Restrict to paths matching a glob (`*.ts`, `src/api/*`) or substring (`components/`).'), + budget_tokens: budgetArg(1500), + }, + handler: (args) => { + const { graph, centrality } = getState() + const query = args.query as string + if (!query) return { text: 'Provide a non-empty `query`.', touchedFiles: [] } + const r = searchContent(graph, { + query, + regex: (args.regex as boolean | undefined) ?? false, + caseSensitive: (args.case_sensitive as boolean | undefined) ?? false, + pathFilter: args.path_filter as string | undefined, + budgetTokens: (args.budget_tokens as number | undefined) ?? 1500, + centrality, + }) + return { text: r.text, touchedFiles: r.fileIds } + }, + } +} + +// --------------------------------------------------------------------------- +// trace_calls — walk the CALL graph ("who calls X" / "what does X call"). Uses +// the call edges the index already builds. Replaces grepping for call sites. +// --------------------------------------------------------------------------- +function traceCallsTool(getState: GetState): ToolDef { + return { + name: 'trace_calls', + description: + 'Trace the call graph from a function/method: callers (who calls it — impact) or callees (what it calls). Multi-hop, ranked by importance, rendered as an anchored tree. Use this instead of grepping for call sites or "who uses this".', + inputShape: { + symbol_name: z.string().describe('The function/method name to trace from.'), + direction: z + .enum(['callers', 'callees', 'both']) + .optional() + .describe('callers = who calls it (default), callees = what it calls, both.'), + depth: z.number().int().min(1).max(6).optional().describe('Call hops to follow (default 3).'), + budget_tokens: budgetArg(1200), + }, + handler: (args) => { + const { graph, centrality } = getState() + const symbolName = args.symbol_name as string + if (!symbolName) return { text: 'Provide a `symbol_name`.', touchedFiles: [] } + const r = traceCalls(graph, { + symbolName, + direction: (args.direction as 'callers' | 'callees' | 'both' | undefined) ?? 'callers', + depth: args.depth as number | undefined, + budgetTokens: (args.budget_tokens as number | undefined) ?? 1200, + centrality, + }) + return { text: r.text, touchedFiles: r.fileIds } + }, + } +} + // --------------------------------------------------------------------------- // get_neighborhood — the local subgraph a senior engineer would point at. // --------------------------------------------------------------------------- @@ -400,6 +482,8 @@ export function buildTools(getState: GetState, deps?: ToolDeps): ToolDef[] { resolveContextTool(getState), skeletonTool(getState), findSymbolTool(getState), + searchCodeTool(getState), + traceCallsTool(getState), neighborhoodTool(getState), dependentsTool(getState), hotspotsTool(getState), diff --git a/ui/components/graph/AtlasView.tsx b/ui/components/graph/AtlasView.tsx index 32c6556..4f1065f 100644 --- a/ui/components/graph/AtlasView.tsx +++ b/ui/components/graph/AtlasView.tsx @@ -24,11 +24,12 @@ export interface AtlasViewProps { embed?: boolean } -const NODE_TYPES: AtlasNodeType[] = ['file', 'function', 'class', 'interface', 'type', 'const'] -const LINK_KINDS: AtlasLinkKind[] = ['imports', 'defines', 'calls'] +const NODE_TYPES: AtlasNodeType[] = ['file', 'function', 'method', 'class', 'interface', 'type', 'const'] +const LINK_KINDS: AtlasLinkKind[] = ['imports', 'defines', 'calls', 'extends', 'implements'] const NODE_LABEL: Record = { file: 'File', function: 'Function', + method: 'Method', class: 'Class', interface: 'Interface', type: 'Type', @@ -47,7 +48,7 @@ const NODE_VERT = ` void main() { vColor = aColor; vec4 mv = modelViewMatrix * vec4(position, 1.0); - gl_PointSize = clamp(size * uScale / max(1.0, -mv.z), 1.5, 64.0); + gl_PointSize = clamp(size * uScale / max(1.0, -mv.z), 1.5, 128.0); gl_Position = projectionMatrix * mv; } ` @@ -273,7 +274,16 @@ function AtlasScene({ atlas, bounds, hiddenTypes, hiddenLinks, focusedFileId, on const colors = new Float32Array(kept.length * 6) for (let i = 0; i < kept.length; i++) { const { s, t, kind } = kept[i]! - const intensity = kind === 'imports' ? 0.5 : kind === 'calls' ? 0.6 : 0.18 + const intensity = + kind === 'imports' + ? 0.5 + : kind === 'calls' + ? 0.6 + : kind === 'extends' + ? 0.75 + : kind === 'implements' + ? 0.7 + : 0.18 const [r, g, b] = toColor(ATLAS_LINK_COLOR[kind], intensity) const o = i * 6 positions[o] = s.x diff --git a/ui/lib/api/types.ts b/ui/lib/api/types.ts index 8194e25..1b23f5f 100644 --- a/ui/lib/api/types.ts +++ b/ui/lib/api/types.ts @@ -64,7 +64,7 @@ export const FileSchema = z.object({ }) export type File = z.infer -export const SymbolKindSchema = z.enum(['function', 'class', 'const', 'type', 'interface']) +export const SymbolKindSchema = z.enum(['function', 'method', 'class', 'const', 'type', 'interface']) export type SymbolKind = z.infer export const SymbolSchema = z.object({ diff --git a/ui/lib/graph/atlas.ts b/ui/lib/graph/atlas.ts index 31e9b9f..3f48359 100644 --- a/ui/lib/graph/atlas.ts +++ b/ui/lib/graph/atlas.ts @@ -1,15 +1,20 @@ // Assemble the "Atlas": the whole codebase as one interconnected node-link -// graph — files + symbols (functions/classes/types/interfaces/consts) wired by -// `defines` (file → symbol) and `imports` (file → file). Node POSITIONS are the -// engine's deterministic ring-by-folder + Barnes-Hut layout (graph.layout); the -// viewer renders them directly — no client-side force simulation. Heuristic -// `calls` (function → function) land here once the core engine extracts them. +// graph — files + symbols (functions/methods/classes/types/interfaces/consts) +// wired by `defines` (file → symbol), `imports` (file → file), `calls` +// (function → function) and `extends`/`implements` (class → class/interface). +// +// Layout is a SOLID BALL: every file gets an even direction on the unit sphere +// (Fibonacci lattice) and a radial distance set by how connected it is — highly +// connected "hub" files sink toward the core, leaf files float to the outer +// shell. Node SIZE is exaggerated by degree, so the load-bearing nodes read as +// big bright cores. Deterministic — same graph, same ball (no Math.random, no +// client-side force simulation). import type { GraphResponse } from '@/lib/api/types' import { shortName } from '@/components/graph/encoding' -export type AtlasNodeType = 'file' | 'function' | 'class' | 'interface' | 'type' | 'const' -export type AtlasLinkKind = 'imports' | 'defines' | 'calls' +export type AtlasNodeType = 'file' | 'function' | 'method' | 'class' | 'interface' | 'type' | 'const' +export type AtlasLinkKind = 'imports' | 'defines' | 'calls' | 'extends' | 'implements' export interface AtlasNode { id: string @@ -17,13 +22,12 @@ export interface AtlasNode { type: AtlasNodeType /** The owning file id (for click-to-focus + filtering). */ fileId: number - /** Render radius (files scale with LOC; symbols are small). */ + /** Render radius — exaggerated by degree for files, by kind for symbols. */ radius: number /** Precomputed world position — the viewer renders this, no client sim. */ x: number y: number - /** Depth — the atlas is a true 3D galaxy: files dome up from the centre and - * symbols orbit their file on a 3D shell. */ + /** Depth — the atlas is a true 3D ball, not a flat disc. */ z: number } @@ -48,6 +52,7 @@ export interface AtlasData { export const ATLAS_NODE_COLOR: Record = { file: '#3b82f6', // blue function: '#22d3ee', // cyan + method: '#2dd4bf', // teal class: '#a855f7', // purple interface: '#c084fc', // violet type: '#94a3b8', // slate @@ -58,6 +63,8 @@ export const ATLAS_LINK_COLOR: Record = { imports: '#3b82f6', defines: '#475569', calls: '#f59e0b', // amber (matches the spotlight convention) + extends: '#ec4899', // pink — inheritance + implements: '#34d399', // green — interface implementation } export function buildAtlas(graph: GraphResponse): AtlasData { @@ -66,58 +73,48 @@ export function buildAtlas(graph: GraphResponse): AtlasData { const nodeCounts: Record = { file: 0, function: 0, + method: 0, class: 0, interface: 0, type: 0, const: 0, } - const linkCounts: Record = { imports: 0, defines: 0, calls: 0 } + const linkCounts: Record = { + imports: 0, + defines: 0, + calls: 0, + extends: 0, + implements: 0, + } const fileNodeId = (id: number) => `f${id}` const symNodeId = (id: number) => `s${id}` // Every node shows up — no capping. The viewer draws the whole codebase with - // WebGL instancing (one Points draw call + one LineSegments draw call), so even - // large repos render as one galaxy. + // WebGL instancing (one Points draw call + one LineSegments draw call). const files = graph.files const fileIds = new Set(files.map((f) => f.id)) const symbols = graph.symbols.filter((s) => fileIds.has(s.file_id)) - // File positions come from the engine's deterministic layout (ring-by-folder + - // Barnes-Hut). Fall back to a golden-angle spiral if no layout was supplied. - const layoutPos = new Map() - for (const ln of graph.layout?.nodes ?? []) layoutPos.set(ln.id, { x: ln.x, y: ln.y }) - const filePos = new Map() - files.forEach((f, i) => { - const p = layoutPos.get(f.id) - if (p) filePos.set(f.id, p) - else { - const a = i * 2.399963 // golden angle - const r = 40 * Math.sqrt(i + 1) - filePos.set(f.id, { x: Math.cos(a) * r, y: Math.sin(a) * r }) - } - }) - - // ---- Lift the flat layout into 3D. Files dome up from the centroid (central, - // load-bearing files rise toward the viewer) with a little deterministic jitter - // for volume; symbols then orbit their file on a 3D shell. Deterministic — same - // graph, same galaxy. ---- - let cx = 0 - let cy = 0 - for (const p of filePos.values()) { - cx += p.x - cy += p.y + // ---- Degree: how connected each file is. Drives BOTH node size and radial + // depth in the ball (hubs are big and pulled to the core). ---- + const fileDegree = new Map() + const bumpDeg = (id: number, n = 1) => { + if (fileIds.has(id)) fileDegree.set(id, (fileDegree.get(id) ?? 0) + n) } - const nf = filePos.size || 1 - cx /= nf - cy /= nf - let maxR = 1 - for (const p of filePos.values()) { - const d = Math.hypot(p.x - cx, p.y - cy) - if (d > maxR) maxR = d + for (const e of graph.edges) { + if (e.edge_kind === 'import' && e.source_kind === 'file' && e.target_kind === 'file') { + bumpDeg(e.source_id) + bumpDeg(e.target_id) + } } + // The symbols a file defines add a little mass — a fat file is a hub too. + for (const s of symbols) bumpDeg(s.file_id, 0.2) + let maxDeg = 1 + for (const d of fileDegree.values()) if (d > maxDeg) maxDeg = d + // Deterministic hash → [-1, 1] from an integer id (no Math.random, so reloads - // don't reshuffle the galaxy). + // don't reshuffle the ball). const jitter = (n: number) => { let h = (n * 2654435761) >>> 0 h ^= h >>> 15 @@ -125,54 +122,76 @@ export function buildAtlas(graph: GraphResponse): AtlasData { h ^= h >>> 13 return ((h >>> 0) / 4294967295) * 2 - 1 } - const Z_BULGE = maxR * 0.7 - const fileZ = new Map() - for (const f of files) { - const p = filePos.get(f.id)! - const rr = Math.hypot(p.x - cx, p.y - cy) / maxR - fileZ.set(f.id, Z_BULGE * (1 - rr * rr) + jitter(f.id) * maxR * 0.14) - } + + // ---- Solid ball. Fibonacci direction on the unit sphere (even angular + // coverage) × a radial distance driven by connectedness: hubs to the core, + // leaves to the shell. ---- + const RADIUS = Math.max(180, 30 * Math.sqrt(Math.max(1, files.length))) + const N = files.length + const filePos = new Map() + files.forEach((f, i) => { + const deg = fileDegree.get(f.id) ?? 0 + const t = Math.min(1, deg / maxDeg) // 0 (leaf) .. 1 (hub) + const yy = N > 1 ? 1 - (i / (N - 1)) * 2 : 0 // 1 .. -1 + const ring = Math.sqrt(Math.max(0, 1 - yy * yy)) + const theta = i * 2.399963 // golden angle + const ux = Math.cos(theta) * ring + const uz = Math.sin(theta) * ring + // Hubs (t→1) pull to the core; leaves (t→0) push to the shell. Jitter adds + // volume so shells don't look like hard rings. + const rr = RADIUS * (0.16 + 0.84 * Math.pow(1 - t, 1.4)) * (0.92 + 0.08 * jitter(f.id)) + filePos.set(f.id, { x: ux * rr, y: yy * rr, z: uz * rr }) + }) for (const f of files) { const p = filePos.get(f.id)! + const deg = fileDegree.get(f.id) ?? 0 nodes.push({ id: fileNodeId(f.id), label: shortName(f.path), type: 'file', fileId: f.id, - radius: Math.max(3, Math.min(9, 3 + Math.sqrt(f.loc) / 6)), + // Exaggerated by connections: leaves ~5, hubs large (shader caps on-screen). + radius: 5 + Math.sqrt(deg) * 4 + Math.min(6, Math.sqrt(f.loc) / 6), x: p.x, y: p.y, - z: fileZ.get(f.id) ?? 0, + z: p.z, }) nodeCounts.file++ } - // Symbols orbit their owning file on a 3D shell (golden-angle azimuth × evenly - // sliced inclination) — deterministic, and tight enough to stay near the file. + // Symbols orbit their owning file on a small 3D shell (golden-angle direction), + // sized by kind so classes/interfaces read larger than their members. + const SYM_RADIUS: Record = { + file: 5, + class: 4.2, + interface: 3.8, + type: 2.8, + function: 2.8, + method: 2.3, + const: 2.1, + } const symOrbit = new Map() const symIds = new Set() for (const s of symbols) { - if (!fileIds.has(s.file_id)) continue const type = s.kind as AtlasNodeType if (!(type in nodeCounts)) continue const fp = filePos.get(s.file_id)! - const fz = fileZ.get(s.file_id) ?? 0 const k = symOrbit.get(s.file_id) ?? 0 symOrbit.set(s.file_id, k + 1) - const ang = k * 2.399963 // golden angle azimuth - const zoff = ((k % 16) / 15) * 2 - 1 // -1..1 inclination band - const ring = Math.sqrt(Math.max(0, 1 - zoff * zoff)) - const orad = 7 + k * 1.1 + const yy = ((k % 16) / 15) * 2 - 1 // -1..1 inclination band + const ring = Math.sqrt(Math.max(0, 1 - yy * yy)) + const ang = k * 2.399963 // golden-angle azimuth + const orad = 9 + k * 1.2 nodes.push({ id: symNodeId(s.id), label: s.name, type, fileId: s.file_id, - radius: 2.4, + radius: SYM_RADIUS[type] ?? 2.4, x: fp.x + Math.cos(ang) * ring * orad, - y: fp.y + Math.sin(ang) * ring * orad, - z: fz + zoff * orad, + y: fp.y + yy * orad, + z: fp.z + Math.sin(ang) * ring * orad, }) symIds.add(s.id) nodeCounts[type]++ @@ -181,7 +200,7 @@ export function buildAtlas(graph: GraphResponse): AtlasData { linkCounts.defines++ } - // Nothing is dropped any more. + // Nothing is dropped. const truncated = false for (const e of graph.edges) { @@ -189,12 +208,20 @@ export function buildAtlas(graph: GraphResponse): AtlasData { if (!fileIds.has(e.source_id) || !fileIds.has(e.target_id)) continue links.push({ source: fileNodeId(e.source_id), target: fileNodeId(e.target_id), kind: 'imports' }) linkCounts.imports++ - } else if (e.edge_kind === 'call' && e.source_kind === 'symbol' && e.target_kind === 'symbol') { - // Only if both symbol nodes are present (they may be dropped under the - // exported-only cap on big repos). + } else if (e.source_kind === 'symbol' && e.target_kind === 'symbol') { + // calls / extends / implements — only if both symbol nodes are present. if (!symIds.has(e.source_id) || !symIds.has(e.target_id)) continue - links.push({ source: symNodeId(e.source_id), target: symNodeId(e.target_id), kind: 'calls' }) - linkCounts.calls++ + const kind: AtlasLinkKind | null = + e.edge_kind === 'call' + ? 'calls' + : e.edge_kind === 'extends' + ? 'extends' + : e.edge_kind === 'implements' + ? 'implements' + : null + if (!kind) continue + links.push({ source: symNodeId(e.source_id), target: symNodeId(e.target_id), kind }) + linkCounts[kind]++ } } diff --git a/viewer/src/components/graph/AtlasView.tsx b/viewer/src/components/graph/AtlasView.tsx index 32c6556..4f1065f 100644 --- a/viewer/src/components/graph/AtlasView.tsx +++ b/viewer/src/components/graph/AtlasView.tsx @@ -24,11 +24,12 @@ export interface AtlasViewProps { embed?: boolean } -const NODE_TYPES: AtlasNodeType[] = ['file', 'function', 'class', 'interface', 'type', 'const'] -const LINK_KINDS: AtlasLinkKind[] = ['imports', 'defines', 'calls'] +const NODE_TYPES: AtlasNodeType[] = ['file', 'function', 'method', 'class', 'interface', 'type', 'const'] +const LINK_KINDS: AtlasLinkKind[] = ['imports', 'defines', 'calls', 'extends', 'implements'] const NODE_LABEL: Record = { file: 'File', function: 'Function', + method: 'Method', class: 'Class', interface: 'Interface', type: 'Type', @@ -47,7 +48,7 @@ const NODE_VERT = ` void main() { vColor = aColor; vec4 mv = modelViewMatrix * vec4(position, 1.0); - gl_PointSize = clamp(size * uScale / max(1.0, -mv.z), 1.5, 64.0); + gl_PointSize = clamp(size * uScale / max(1.0, -mv.z), 1.5, 128.0); gl_Position = projectionMatrix * mv; } ` @@ -273,7 +274,16 @@ function AtlasScene({ atlas, bounds, hiddenTypes, hiddenLinks, focusedFileId, on const colors = new Float32Array(kept.length * 6) for (let i = 0; i < kept.length; i++) { const { s, t, kind } = kept[i]! - const intensity = kind === 'imports' ? 0.5 : kind === 'calls' ? 0.6 : 0.18 + const intensity = + kind === 'imports' + ? 0.5 + : kind === 'calls' + ? 0.6 + : kind === 'extends' + ? 0.75 + : kind === 'implements' + ? 0.7 + : 0.18 const [r, g, b] = toColor(ATLAS_LINK_COLOR[kind], intensity) const o = i * 6 positions[o] = s.x diff --git a/viewer/src/lib/api/types.ts b/viewer/src/lib/api/types.ts index 8194e25..1b23f5f 100644 --- a/viewer/src/lib/api/types.ts +++ b/viewer/src/lib/api/types.ts @@ -64,7 +64,7 @@ export const FileSchema = z.object({ }) export type File = z.infer -export const SymbolKindSchema = z.enum(['function', 'class', 'const', 'type', 'interface']) +export const SymbolKindSchema = z.enum(['function', 'method', 'class', 'const', 'type', 'interface']) export type SymbolKind = z.infer export const SymbolSchema = z.object({ diff --git a/viewer/src/lib/graph/atlas.ts b/viewer/src/lib/graph/atlas.ts index 31e9b9f..3f48359 100644 --- a/viewer/src/lib/graph/atlas.ts +++ b/viewer/src/lib/graph/atlas.ts @@ -1,15 +1,20 @@ // Assemble the "Atlas": the whole codebase as one interconnected node-link -// graph — files + symbols (functions/classes/types/interfaces/consts) wired by -// `defines` (file → symbol) and `imports` (file → file). Node POSITIONS are the -// engine's deterministic ring-by-folder + Barnes-Hut layout (graph.layout); the -// viewer renders them directly — no client-side force simulation. Heuristic -// `calls` (function → function) land here once the core engine extracts them. +// graph — files + symbols (functions/methods/classes/types/interfaces/consts) +// wired by `defines` (file → symbol), `imports` (file → file), `calls` +// (function → function) and `extends`/`implements` (class → class/interface). +// +// Layout is a SOLID BALL: every file gets an even direction on the unit sphere +// (Fibonacci lattice) and a radial distance set by how connected it is — highly +// connected "hub" files sink toward the core, leaf files float to the outer +// shell. Node SIZE is exaggerated by degree, so the load-bearing nodes read as +// big bright cores. Deterministic — same graph, same ball (no Math.random, no +// client-side force simulation). import type { GraphResponse } from '@/lib/api/types' import { shortName } from '@/components/graph/encoding' -export type AtlasNodeType = 'file' | 'function' | 'class' | 'interface' | 'type' | 'const' -export type AtlasLinkKind = 'imports' | 'defines' | 'calls' +export type AtlasNodeType = 'file' | 'function' | 'method' | 'class' | 'interface' | 'type' | 'const' +export type AtlasLinkKind = 'imports' | 'defines' | 'calls' | 'extends' | 'implements' export interface AtlasNode { id: string @@ -17,13 +22,12 @@ export interface AtlasNode { type: AtlasNodeType /** The owning file id (for click-to-focus + filtering). */ fileId: number - /** Render radius (files scale with LOC; symbols are small). */ + /** Render radius — exaggerated by degree for files, by kind for symbols. */ radius: number /** Precomputed world position — the viewer renders this, no client sim. */ x: number y: number - /** Depth — the atlas is a true 3D galaxy: files dome up from the centre and - * symbols orbit their file on a 3D shell. */ + /** Depth — the atlas is a true 3D ball, not a flat disc. */ z: number } @@ -48,6 +52,7 @@ export interface AtlasData { export const ATLAS_NODE_COLOR: Record = { file: '#3b82f6', // blue function: '#22d3ee', // cyan + method: '#2dd4bf', // teal class: '#a855f7', // purple interface: '#c084fc', // violet type: '#94a3b8', // slate @@ -58,6 +63,8 @@ export const ATLAS_LINK_COLOR: Record = { imports: '#3b82f6', defines: '#475569', calls: '#f59e0b', // amber (matches the spotlight convention) + extends: '#ec4899', // pink — inheritance + implements: '#34d399', // green — interface implementation } export function buildAtlas(graph: GraphResponse): AtlasData { @@ -66,58 +73,48 @@ export function buildAtlas(graph: GraphResponse): AtlasData { const nodeCounts: Record = { file: 0, function: 0, + method: 0, class: 0, interface: 0, type: 0, const: 0, } - const linkCounts: Record = { imports: 0, defines: 0, calls: 0 } + const linkCounts: Record = { + imports: 0, + defines: 0, + calls: 0, + extends: 0, + implements: 0, + } const fileNodeId = (id: number) => `f${id}` const symNodeId = (id: number) => `s${id}` // Every node shows up — no capping. The viewer draws the whole codebase with - // WebGL instancing (one Points draw call + one LineSegments draw call), so even - // large repos render as one galaxy. + // WebGL instancing (one Points draw call + one LineSegments draw call). const files = graph.files const fileIds = new Set(files.map((f) => f.id)) const symbols = graph.symbols.filter((s) => fileIds.has(s.file_id)) - // File positions come from the engine's deterministic layout (ring-by-folder + - // Barnes-Hut). Fall back to a golden-angle spiral if no layout was supplied. - const layoutPos = new Map() - for (const ln of graph.layout?.nodes ?? []) layoutPos.set(ln.id, { x: ln.x, y: ln.y }) - const filePos = new Map() - files.forEach((f, i) => { - const p = layoutPos.get(f.id) - if (p) filePos.set(f.id, p) - else { - const a = i * 2.399963 // golden angle - const r = 40 * Math.sqrt(i + 1) - filePos.set(f.id, { x: Math.cos(a) * r, y: Math.sin(a) * r }) - } - }) - - // ---- Lift the flat layout into 3D. Files dome up from the centroid (central, - // load-bearing files rise toward the viewer) with a little deterministic jitter - // for volume; symbols then orbit their file on a 3D shell. Deterministic — same - // graph, same galaxy. ---- - let cx = 0 - let cy = 0 - for (const p of filePos.values()) { - cx += p.x - cy += p.y + // ---- Degree: how connected each file is. Drives BOTH node size and radial + // depth in the ball (hubs are big and pulled to the core). ---- + const fileDegree = new Map() + const bumpDeg = (id: number, n = 1) => { + if (fileIds.has(id)) fileDegree.set(id, (fileDegree.get(id) ?? 0) + n) } - const nf = filePos.size || 1 - cx /= nf - cy /= nf - let maxR = 1 - for (const p of filePos.values()) { - const d = Math.hypot(p.x - cx, p.y - cy) - if (d > maxR) maxR = d + for (const e of graph.edges) { + if (e.edge_kind === 'import' && e.source_kind === 'file' && e.target_kind === 'file') { + bumpDeg(e.source_id) + bumpDeg(e.target_id) + } } + // The symbols a file defines add a little mass — a fat file is a hub too. + for (const s of symbols) bumpDeg(s.file_id, 0.2) + let maxDeg = 1 + for (const d of fileDegree.values()) if (d > maxDeg) maxDeg = d + // Deterministic hash → [-1, 1] from an integer id (no Math.random, so reloads - // don't reshuffle the galaxy). + // don't reshuffle the ball). const jitter = (n: number) => { let h = (n * 2654435761) >>> 0 h ^= h >>> 15 @@ -125,54 +122,76 @@ export function buildAtlas(graph: GraphResponse): AtlasData { h ^= h >>> 13 return ((h >>> 0) / 4294967295) * 2 - 1 } - const Z_BULGE = maxR * 0.7 - const fileZ = new Map() - for (const f of files) { - const p = filePos.get(f.id)! - const rr = Math.hypot(p.x - cx, p.y - cy) / maxR - fileZ.set(f.id, Z_BULGE * (1 - rr * rr) + jitter(f.id) * maxR * 0.14) - } + + // ---- Solid ball. Fibonacci direction on the unit sphere (even angular + // coverage) × a radial distance driven by connectedness: hubs to the core, + // leaves to the shell. ---- + const RADIUS = Math.max(180, 30 * Math.sqrt(Math.max(1, files.length))) + const N = files.length + const filePos = new Map() + files.forEach((f, i) => { + const deg = fileDegree.get(f.id) ?? 0 + const t = Math.min(1, deg / maxDeg) // 0 (leaf) .. 1 (hub) + const yy = N > 1 ? 1 - (i / (N - 1)) * 2 : 0 // 1 .. -1 + const ring = Math.sqrt(Math.max(0, 1 - yy * yy)) + const theta = i * 2.399963 // golden angle + const ux = Math.cos(theta) * ring + const uz = Math.sin(theta) * ring + // Hubs (t→1) pull to the core; leaves (t→0) push to the shell. Jitter adds + // volume so shells don't look like hard rings. + const rr = RADIUS * (0.16 + 0.84 * Math.pow(1 - t, 1.4)) * (0.92 + 0.08 * jitter(f.id)) + filePos.set(f.id, { x: ux * rr, y: yy * rr, z: uz * rr }) + }) for (const f of files) { const p = filePos.get(f.id)! + const deg = fileDegree.get(f.id) ?? 0 nodes.push({ id: fileNodeId(f.id), label: shortName(f.path), type: 'file', fileId: f.id, - radius: Math.max(3, Math.min(9, 3 + Math.sqrt(f.loc) / 6)), + // Exaggerated by connections: leaves ~5, hubs large (shader caps on-screen). + radius: 5 + Math.sqrt(deg) * 4 + Math.min(6, Math.sqrt(f.loc) / 6), x: p.x, y: p.y, - z: fileZ.get(f.id) ?? 0, + z: p.z, }) nodeCounts.file++ } - // Symbols orbit their owning file on a 3D shell (golden-angle azimuth × evenly - // sliced inclination) — deterministic, and tight enough to stay near the file. + // Symbols orbit their owning file on a small 3D shell (golden-angle direction), + // sized by kind so classes/interfaces read larger than their members. + const SYM_RADIUS: Record = { + file: 5, + class: 4.2, + interface: 3.8, + type: 2.8, + function: 2.8, + method: 2.3, + const: 2.1, + } const symOrbit = new Map() const symIds = new Set() for (const s of symbols) { - if (!fileIds.has(s.file_id)) continue const type = s.kind as AtlasNodeType if (!(type in nodeCounts)) continue const fp = filePos.get(s.file_id)! - const fz = fileZ.get(s.file_id) ?? 0 const k = symOrbit.get(s.file_id) ?? 0 symOrbit.set(s.file_id, k + 1) - const ang = k * 2.399963 // golden angle azimuth - const zoff = ((k % 16) / 15) * 2 - 1 // -1..1 inclination band - const ring = Math.sqrt(Math.max(0, 1 - zoff * zoff)) - const orad = 7 + k * 1.1 + const yy = ((k % 16) / 15) * 2 - 1 // -1..1 inclination band + const ring = Math.sqrt(Math.max(0, 1 - yy * yy)) + const ang = k * 2.399963 // golden-angle azimuth + const orad = 9 + k * 1.2 nodes.push({ id: symNodeId(s.id), label: s.name, type, fileId: s.file_id, - radius: 2.4, + radius: SYM_RADIUS[type] ?? 2.4, x: fp.x + Math.cos(ang) * ring * orad, - y: fp.y + Math.sin(ang) * ring * orad, - z: fz + zoff * orad, + y: fp.y + yy * orad, + z: fp.z + Math.sin(ang) * ring * orad, }) symIds.add(s.id) nodeCounts[type]++ @@ -181,7 +200,7 @@ export function buildAtlas(graph: GraphResponse): AtlasData { linkCounts.defines++ } - // Nothing is dropped any more. + // Nothing is dropped. const truncated = false for (const e of graph.edges) { @@ -189,12 +208,20 @@ export function buildAtlas(graph: GraphResponse): AtlasData { if (!fileIds.has(e.source_id) || !fileIds.has(e.target_id)) continue links.push({ source: fileNodeId(e.source_id), target: fileNodeId(e.target_id), kind: 'imports' }) linkCounts.imports++ - } else if (e.edge_kind === 'call' && e.source_kind === 'symbol' && e.target_kind === 'symbol') { - // Only if both symbol nodes are present (they may be dropped under the - // exported-only cap on big repos). + } else if (e.source_kind === 'symbol' && e.target_kind === 'symbol') { + // calls / extends / implements — only if both symbol nodes are present. if (!symIds.has(e.source_id) || !symIds.has(e.target_id)) continue - links.push({ source: symNodeId(e.source_id), target: symNodeId(e.target_id), kind: 'calls' }) - linkCounts.calls++ + const kind: AtlasLinkKind | null = + e.edge_kind === 'call' + ? 'calls' + : e.edge_kind === 'extends' + ? 'extends' + : e.edge_kind === 'implements' + ? 'implements' + : null + if (!kind) continue + links.push({ source: symNodeId(e.source_id), target: symNodeId(e.target_id), kind }) + linkCounts[kind]++ } }