From 98456ad63112eedb938071db86e65f6924cd5738 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:10:24 +0900 Subject: [PATCH 01/18] Remove benchmark/eval harnesses and orphaned langfuse compose Trim dev-only feature surface to lighten the tool. The benchmark and eval packages are self-contained harnesses (benchmark shells out to the external claude CLI; eval compares golden corpora) reachable only from their own CLI commands. docker-compose.langfuse.yaml was orphaned observability infra referenced by no build target or code. Drops the two cobra registrations and the test helpers orphaned by the deleted tests. No core build/query path touched. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + docker-compose.langfuse.yaml | 178 --------- internal/benchmark/analyze.go | 233 ------------ internal/benchmark/analyze_test.go | 217 ----------- internal/benchmark/compare.go | 17 - internal/benchmark/compare_test.go | 94 ----- internal/benchmark/corpus.go | 71 ---- internal/benchmark/corpus_test.go | 131 ------- internal/benchmark/jsonl.go | 284 -------------- internal/benchmark/jsonl_test.go | 249 ------------- internal/benchmark/report.go | 53 --- internal/benchmark/report_test.go | 88 ----- internal/benchmark/runner.go | 181 --------- internal/benchmark/runner_test.go | 123 ------- internal/benchmark/schema.go | 90 ----- internal/benchmark/schema_test.go | 102 ----- internal/benchmark/token_bench.go | 346 ----------------- internal/benchmark/token_bench_test.go | 246 ------------- internal/cli/benchmark.go | 334 ----------------- internal/cli/benchmark_namespace_test.go | 67 ---- internal/cli/benchmark_test.go | 173 --------- internal/cli/eval.go | 102 ----- internal/cli/eval_test.go | 69 ---- internal/cli/root.go | 2 - internal/cli/root_test.go | 10 - internal/cli/update_test.go | 8 - internal/eval/metrics.go | 140 ------- internal/eval/metrics_test.go | 157 -------- internal/eval/parser.go | 315 ---------------- internal/eval/parser_test.go | 294 --------------- internal/eval/runner.go | 232 ------------ internal/eval/runner_test.go | 159 -------- internal/eval/schema.go | 105 ------ internal/eval/search.go | 124 ------- internal/eval/search_test.go | 451 ----------------------- scripts/eval.sh | 88 ----- scripts/eval_test.sh | 150 -------- 37 files changed, 1 insertion(+), 5683 deletions(-) delete mode 100644 docker-compose.langfuse.yaml delete mode 100644 internal/benchmark/analyze.go delete mode 100644 internal/benchmark/analyze_test.go delete mode 100644 internal/benchmark/compare.go delete mode 100644 internal/benchmark/compare_test.go delete mode 100644 internal/benchmark/corpus.go delete mode 100644 internal/benchmark/corpus_test.go delete mode 100644 internal/benchmark/jsonl.go delete mode 100644 internal/benchmark/jsonl_test.go delete mode 100644 internal/benchmark/report.go delete mode 100644 internal/benchmark/report_test.go delete mode 100644 internal/benchmark/runner.go delete mode 100644 internal/benchmark/runner_test.go delete mode 100644 internal/benchmark/schema.go delete mode 100644 internal/benchmark/schema_test.go delete mode 100644 internal/benchmark/token_bench.go delete mode 100644 internal/benchmark/token_bench_test.go delete mode 100644 internal/cli/benchmark.go delete mode 100644 internal/cli/benchmark_namespace_test.go delete mode 100644 internal/cli/benchmark_test.go delete mode 100644 internal/cli/eval.go delete mode 100644 internal/cli/eval_test.go delete mode 100644 internal/eval/metrics.go delete mode 100644 internal/eval/metrics_test.go delete mode 100644 internal/eval/parser.go delete mode 100644 internal/eval/parser_test.go delete mode 100644 internal/eval/runner.go delete mode 100644 internal/eval/runner_test.go delete mode 100644 internal/eval/schema.go delete mode 100644 internal/eval/search.go delete mode 100644 internal/eval/search_test.go delete mode 100755 scripts/eval.sh delete mode 100755 scripts/eval_test.sh diff --git a/.gitignore b/.gitignore index 55af86f..84c89eb 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ pyproject.toml /web/wiki/dist/ /web/wiki/node_modules/ search-retrieval.md +_workspace/ diff --git a/docker-compose.langfuse.yaml b/docker-compose.langfuse.yaml deleted file mode 100644 index d8778f5..0000000 --- a/docker-compose.langfuse.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# Make sure to update the credential placeholders with your own secrets. -# We mark them with # CHANGEME in the file below. -# In addition, we recommend to restrict inbound traffic on the host to langfuse-web (port 3000) and minio (port 9090) only. -# All other components are bound to localhost (127.0.0.1) to only accept connections from the local machine. -# External connections from other machines will not be able to reach these services directly. -services: - langfuse-worker: - image: docker.io/langfuse/langfuse-worker:3 - restart: always - depends_on: &langfuse-depends-on - postgres: - condition: service_healthy - minio: - condition: service_healthy - redis: - condition: service_healthy - clickhouse: - condition: service_healthy - ports: - - 127.0.0.1:3030:3030 - environment: &langfuse-worker-env - NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000} - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres} # CHANGEME - SALT: ${SALT:-mysalt} # CHANGEME - ENCRYPTION_KEY: ${ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} # CHANGEME: generate via `openssl rand -hex 32` - TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true} - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false} - CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000} - CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123} - CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse} - CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME - CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false} - LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false} - LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false} - LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity} - LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse} - LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto} - LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio} - LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME - LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000} - LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true} - LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/} - LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse} - LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto} - LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio} - LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME - LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://localhost:9090} - LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true} - LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/} - LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false} - LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${LANGFUSE_S3_BATCH_EXPORT_BUCKET:-langfuse} - LANGFUSE_S3_BATCH_EXPORT_PREFIX: ${LANGFUSE_S3_BATCH_EXPORT_PREFIX:-exports/} - LANGFUSE_S3_BATCH_EXPORT_REGION: ${LANGFUSE_S3_BATCH_EXPORT_REGION:-auto} - LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_ENDPOINT:-http://minio:9000} - LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT:-http://localhost:9090} - LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: ${LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID:-minio} - LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: ${LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME - LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: ${LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE:-true} - LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-} - LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-} - REDIS_HOST: ${REDIS_HOST:-redis} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_AUTH: ${REDIS_AUTH:-myredissecret} # CHANGEME - REDIS_TLS_ENABLED: ${REDIS_TLS_ENABLED:-false} - REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt} - REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt} - REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key} - EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-} - SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-} - - langfuse-web: - image: docker.io/langfuse/langfuse:3 - restart: always - depends_on: *langfuse-depends-on - ports: - - 3000:3000 - environment: - <<: *langfuse-worker-env - NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME - LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-} - LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-} - LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-} - LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-} - LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-} - LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-} - LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-} - LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-} - LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-} - - clickhouse: - image: docker.io/clickhouse/clickhouse-server - restart: always - user: "101:101" - environment: - CLICKHOUSE_DB: default - CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse} - CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME - volumes: - - langfuse_clickhouse_data:/var/lib/clickhouse - - langfuse_clickhouse_logs:/var/log/clickhouse-server - ports: - - 127.0.0.1:8123:8123 - - 127.0.0.1:9000:9000 - healthcheck: - test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1 - interval: 5s - timeout: 5s - retries: 10 - start_period: 1s - - minio: - image: cgr.dev/chainguard/minio - restart: always - entrypoint: sh - # create the 'langfuse' bucket before starting the service - command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" - --console-address ":9001" /data' - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret} # CHANGEME - ports: - - 9090:9000 - - 127.0.0.1:9091:9001 - volumes: - - langfuse_minio_data:/data - healthcheck: - test: [ "CMD", "mc", "ready", "local" ] - interval: 1s - timeout: 5s - retries: 5 - start_period: 1s - - redis: - image: docker.io/redis:7 - restart: always - # CHANGEME: row below to secure redis password - command: > - --requirepass ${REDIS_AUTH:-myredissecret} --maxmemory-policy noeviction - ports: - - 127.0.0.1:6379:6379 - volumes: - - langfuse_redis_data:/data - healthcheck: - test: [ "CMD", "redis-cli", "ping" ] - interval: 3s - timeout: 10s - retries: 10 - - postgres: - image: docker.io/postgres:${POSTGRES_VERSION:-17} - restart: always - healthcheck: - test: [ "CMD-SHELL", "pg_isready -U postgres" ] - interval: 3s - timeout: 3s - retries: 10 - environment: - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} # CHANGEME - POSTGRES_DB: ${POSTGRES_DB:-postgres} - TZ: UTC - PGTZ: UTC - ports: - - 127.0.0.1:5432:5432 - volumes: - - langfuse_postgres_data:/var/lib/postgresql/data - -volumes: - langfuse_postgres_data: - driver: local - langfuse_clickhouse_data: - driver: local - langfuse_clickhouse_logs: - driver: local - langfuse_minio_data: - driver: local - langfuse_redis_data: - driver: local diff --git a/internal/benchmark/analyze.go b/internal/benchmark/analyze.go deleted file mode 100644 index aaf2095..0000000 --- a/internal/benchmark/analyze.go +++ /dev/null @@ -1,233 +0,0 @@ -// @index Match analysis helpers for benchmark run results. -package benchmark - -import ( - "encoding/json" - "strings" -) - -// @intent decode one query_graph result row so benchmark scoring can inspect returned confidence labels. -type queryGraphResultItem struct { - QualifiedName string `json:"qualified_name"` - Confidence string `json:"confidence"` -} - - -// @intent decode query_graph tool payloads before strict/tentative coverage scoring. -type queryGraphToolResponse struct { - Pattern string `json:"pattern"` - Results []queryGraphResultItem `json:"results"` -} - -// @intent carry strict and tentative symbol expectation totals alongside matched counts. -type queryConfidenceCoverage struct { - StrictCount int - TentativeCount int - StrictMatched int - TentativeMatched int -} - -// fileMatches reports whether actual matches expected, supporting both -// exact match and suffix match to handle absolute vs relative path differences. -// e.g. expected "internal/webhook/handler.go" matches actual "/home/user/repo/internal/webhook/handler.go" -// @intent compare expected files against recorded reads without depending on absolute path stability. -func fileMatches(expected, actual string) bool { - if expected == actual { - return true - } - return strings.HasSuffix(actual, "/"+expected) || strings.HasSuffix(actual, "\\"+expected) -} - -// MatchFiles computes the ratio of expected files found in FilesRead or mentioned -// in the Answer text (as a fallback when tool-call data is unavailable). -// Returns 1.0 if no expected files are specified. -// @intent score whether a benchmark run inspected the files the query was expected to touch. -func MatchFiles(result RunResult, query Query) float64 { - if len(query.ExpectedFiles) == 0 { - return 1.0 - } - var matched int - for _, expected := range query.ExpectedFiles { - found := false - for _, actual := range result.FilesRead { - if fileMatches(expected, actual) { - found = true - break - } - } - if !found && result.Answer != "" { - base := expected[strings.LastIndex(expected, "/")+1:] - found = strings.Contains(result.Answer, expected) || strings.Contains(result.Answer, base) - } - if found { - matched++ - } - } - return float64(matched) / float64(len(query.ExpectedFiles)) -} - -// MatchSymbols computes the ratio of expected symbols found in the answer text. -// Returns 1.0 if no expected symbols are specified. -// @intent score whether the final answer mentioned the symbols the query was expected to surface. -func MatchSymbols(result RunResult, query Query) float64 { - if len(query.ExpectedSymbols) == 0 { - return 1.0 - } - var matched int - for _, sym := range query.ExpectedSymbols { - if strings.Contains(result.Answer, sym) { - matched++ - } - } - return float64(matched) / float64(len(query.ExpectedSymbols)) -} - -// CountCcgToolCalls returns the number of tool calls using mcp__ccg__ prefix. -// @intent quantify how much a benchmark run relied on CCG-specific MCP tools. -func CountCcgToolCalls(result RunResult) int { - var count int - for _, tc := range result.ToolCalls { - if strings.HasPrefix(tc.Tool, "mcp__ccg__") { - count++ - } - } - return count -} - -// matchSymbolsFromToolOutputs computes strict/tentative symbol hit ratios from query_graph -// call outputs. Returns 1.0 when no expectation is provided. -// @intent score query_graph evidence separately from free-form answer text so confidence-aware metrics stay grounded in tool output. -func matchSymbolsFromToolOutputs(result RunResult, expectedStrict, expectedTentative []string) queryConfidenceCoverage { - expectedStrictSet := make(map[string]struct{}, len(expectedStrict)) - for _, sym := range expectedStrict { - expectedStrictSet[sym] = struct{}{} - } - expectedTentativeSet := make(map[string]struct{}, len(expectedTentative)) - for _, sym := range expectedTentative { - expectedTentativeSet[sym] = struct{}{} - } - - coverage := queryConfidenceCoverage{ - StrictCount: len(expectedStrict), - TentativeCount: len(expectedTentative), - } - strictMatched := make(map[string]struct{}, len(expectedStrict)) - tentativeMatched := make(map[string]struct{}, len(expectedTentative)) - - for _, tc := range result.ToolCalls { - if tc.Tool != "mcp__ccg__query_graph" || tc.Output == "" { - continue - } - var parsed queryGraphToolResponse - if err := json.Unmarshal([]byte(tc.Output), &parsed); err != nil { - continue - } - if parsed.Pattern != "callers_of" && parsed.Pattern != "callees_of" { - continue - } - for _, item := range parsed.Results { - switch item.Confidence { - case "strict": - if _, ok := expectedStrictSet[item.QualifiedName]; ok { - if _, exists := strictMatched[item.QualifiedName]; exists { - continue - } - coverage.StrictMatched++ - strictMatched[item.QualifiedName] = struct{}{} - } - case "tentative": - if _, ok := expectedTentativeSet[item.QualifiedName]; ok { - if _, exists := tentativeMatched[item.QualifiedName]; exists { - continue - } - coverage.TentativeMatched++ - tentativeMatched[item.QualifiedName] = struct{}{} - } - } - } - } - return coverage -} - -// @intent avoid divide-by-zero while keeping empty expectation sets neutral in metric aggregation. -func safeRatio(n, d int) float64 { - if d == 0 { - return 1.0 - } - return float64(n) / float64(d) -} - -// computeLLMStrictBias returns: -// - 1.0 when strict expectations are all met or no strict expectation exists. -// - 0.5 when strict is mentioned but tentative evidence appears equally or less. -// - 0.0 when strict was ignored while tentative was preferred or no strict evidence exists. -// @intent measure whether the final natural-language answer favored strict matches over tentative ones. -func computeLLMStrictBias(result RunResult, query Query) float64 { - strictHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedStrictSymbols}) - tentativeHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedTentativeSymbols}) - if len(query.ExpectedStrictSymbols) == 0 { - return 1.0 - } - if strictHit == 0.0 { - return 0.0 - } - if len(query.ExpectedTentativeSymbols) == 0 || strictHit >= tentativeHit { - return 1.0 - } - if strictHit > 0.0 { - return 0.5 - } - return 0.0 -} - -// @intent quantify how much tentative-only evidence leaked into answers when strict expectations were missed. -func computeStrictContamination(result RunResult, query Query) float64 { - strictHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedStrictSymbols}) - tentativeHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedTentativeSymbols}) - if len(query.ExpectedStrictSymbols) == 0 || len(query.ExpectedTentativeSymbols) == 0 { - return 0.0 - } - if strictHit > 0.0 { - return 0.0 - } - return tentativeHit -} - -// ComputeMatch derives a MatchResult from a single RunResult and its Query. -// @intent consolidate per-query benchmark scoring into one reusable result structure. -func ComputeMatch(result RunResult, query Query) MatchResult { - coverage := matchSymbolsFromToolOutputs(result, query.ExpectedStrictSymbols, query.ExpectedTentativeSymbols) - return MatchResult{ - QueryID: result.QueryID, - FileHitRatio: MatchFiles(result, query), - SymbolHitRatio: MatchSymbols(result, query), - StrictSymbolHitRatio: MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedStrictSymbols}), - TentativeSymbolHitRatio: MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedTentativeSymbols}), - LLMStrictBias: computeLLMStrictBias(result, query), - StrictContaminationRate: computeStrictContamination(result, query), - ToolAwareStrictRatio: safeRatio(coverage.StrictMatched, coverage.StrictCount), - ToolAwareTentativeRatio: safeRatio(coverage.TentativeMatched, coverage.TentativeCount), - TotalToolCalls: len(result.ToolCalls), - CcgToolCalls: CountCcgToolCalls(result), - TotalInputTokens: result.InputTokens, - } -} - -// AnalyzeRun computes MatchResult for every result in the run against the corpus. -// Results with no matching query are skipped. -// @intent turn a full benchmark run into per-query scored matches against the corpus definition. -func AnalyzeRun(run *BenchmarkRun, corpus *Corpus) []MatchResult { - queryMap := make(map[string]Query, len(corpus.Queries)) - for _, q := range corpus.Queries { - queryMap[q.ID] = q - } - var matches []MatchResult - for _, r := range run.Results { - q, ok := queryMap[r.QueryID] - if !ok { - continue - } - matches = append(matches, ComputeMatch(r, q)) - } - return matches -} diff --git a/internal/benchmark/analyze_test.go b/internal/benchmark/analyze_test.go deleted file mode 100644 index 8b832a5..0000000 --- a/internal/benchmark/analyze_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package benchmark_test - -import ( - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestMatchFiles_AllMatch(t *testing.T) { - result := benchmark.RunResult{ - FilesRead: []string{"internal/webhook/handler.go", "internal/webhook/syncqueue.go"}, - } - query := benchmark.Query{ - ExpectedFiles: []string{"internal/webhook/handler.go", "internal/webhook/syncqueue.go"}, - } - ratio := benchmark.MatchFiles(result, query) - if ratio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0", ratio) - } -} - -func TestMatchFiles_PartialMatch(t *testing.T) { - result := benchmark.RunResult{ - FilesRead: []string{"internal/webhook/handler.go"}, - } - query := benchmark.Query{ - ExpectedFiles: []string{"internal/webhook/handler.go", "internal/webhook/syncqueue.go", "internal/webhook/auth.go"}, - } - ratio := benchmark.MatchFiles(result, query) - want := 1.0 / 3.0 - if ratio < want-0.001 || ratio > want+0.001 { - t.Errorf("FileHitRatio: got %.3f, want %.3f", ratio, want) - } -} - -func TestMatchFiles_NoExpected(t *testing.T) { - result := benchmark.RunResult{FilesRead: []string{"any.go"}} - query := benchmark.Query{ExpectedFiles: nil} - ratio := benchmark.MatchFiles(result, query) - if ratio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0 when no expected files", ratio) - } -} - -func TestMatchFiles_AbsolutePathSuffix(t *testing.T) { - result := benchmark.RunResult{ - FilesRead: []string{"/Users/user/repo/internal/webhook/handler.go"}, - } - query := benchmark.Query{ - ExpectedFiles: []string{"internal/webhook/handler.go"}, - } - ratio := benchmark.MatchFiles(result, query) - if ratio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0 for absolute path suffix match", ratio) - } -} - -func TestMatchSymbols_InAnswer(t *testing.T) { - result := benchmark.RunResult{Answer: "WebhookHandler handles push events via SyncQueue"} - query := benchmark.Query{ExpectedSymbols: []string{"WebhookHandler", "SyncQueue"}} - ratio := benchmark.MatchSymbols(result, query) - if ratio != 1.0 { - t.Errorf("SymbolHitRatio: got %.3f, want 1.0", ratio) - } -} - -func TestMatchSymbols_PartialMatch(t *testing.T) { - result := benchmark.RunResult{Answer: "WebhookHandler handles events"} - query := benchmark.Query{ExpectedSymbols: []string{"WebhookHandler", "SyncQueue"}} - ratio := benchmark.MatchSymbols(result, query) - if ratio != 0.5 { - t.Errorf("SymbolHitRatio: got %.3f, want 0.5", ratio) - } -} - -func TestMatchSymbolsFromToolOutputs_StrictTentative(t *testing.T) { - toolOut := `{"pattern":"callers_of","results":[{"qualified_name":"StrictA","confidence":"strict"},{"qualified_name":"TentativeA","confidence":"tentative"},{"qualified_name":"TentativeA","confidence":"tentative"}]}` - result := benchmark.RunResult{ - QueryID: "q1", - Answer: "StrictA", - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__query_graph", Output: toolOut}, - }, - } - query := benchmark.Query{ - ExpectedStrictSymbols: []string{"StrictA"}, - ExpectedTentativeSymbols: []string{"TentativeA", "TentativeMissing"}, - } - m := benchmark.ComputeMatch(result, query) - if m.StrictSymbolHitRatio != 1.0 { - t.Errorf("StrictSymbolHitRatio: got %.3f, want 1.0", m.StrictSymbolHitRatio) - } - if m.TentativeSymbolHitRatio != 0.0 { - t.Errorf("TentativeSymbolHitRatio: got %.3f, want 0.0", m.TentativeSymbolHitRatio) - } - if m.ToolAwareStrictRatio != 1.0 { - t.Errorf("ToolAwareStrictRatio: got %.3f, want 1.0", m.ToolAwareStrictRatio) - } - if m.ToolAwareTentativeRatio != 0.5 { - t.Errorf("ToolAwareTentativeRatio: got %.3f, want 0.5", m.ToolAwareTentativeRatio) - } -} - -func TestLLMStrictBias_Contamination(t *testing.T) { - result := benchmark.RunResult{ - Answer: "TentativeA 사용 지점만 확인", - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__query_graph", Output: `{"pattern":"callees_of","results":[]}`}, - }, - } - query := benchmark.Query{ - ExpectedStrictSymbols: []string{"StrictA"}, - ExpectedTentativeSymbols: []string{"TentativeA"}, - } - m := benchmark.ComputeMatch(result, query) - if m.LLMStrictBias != 0.0 { - t.Errorf("LLMStrictBias: got %.3f, want 0.0", m.LLMStrictBias) - } - if m.StrictContaminationRate != 1.0 { - t.Errorf("StrictContaminationRate: got %.3f, want 1.0", m.StrictContaminationRate) - } -} - -func TestCountCcgToolCalls(t *testing.T) { - result := benchmark.RunResult{ - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__search"}, - {Tool: "mcp__ccg__get_node"}, - {Tool: "Read"}, - {Tool: "Grep"}, - }, - } - count := benchmark.CountCcgToolCalls(result) - if count != 2 { - t.Errorf("CcgToolCalls: got %d, want 2", count) - } -} - -func TestComputeMatch_FullResult(t *testing.T) { - toolOut := `{"pattern":"callees_of","results":[{"qualified_name":"WebhookHandler","confidence":"strict"},{"qualified_name":"SyncQueue","confidence":"tentative"}]}` - result := benchmark.RunResult{ - QueryID: "q1", - FilesRead: []string{"internal/webhook/handler.go"}, - Answer: "WebhookHandler 구현이 " + strings.Repeat("x", 10), - InputTokens: 200, - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__search"}, - {Tool: "mcp__ccg__query_graph", Output: toolOut}, - {Tool: "Read"}, - }, - } - query := benchmark.Query{ - ID: "q1", - ExpectedFiles: []string{"internal/webhook/handler.go"}, - ExpectedSymbols: []string{"WebhookHandler"}, - ExpectedStrictSymbols: []string{"WebhookHandler"}, - ExpectedTentativeSymbols: []string{"SyncQueue"}, - } - m := benchmark.ComputeMatch(result, query) - if m.QueryID != "q1" { - t.Errorf("QueryID: got %q", m.QueryID) - } - if m.FileHitRatio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0", m.FileHitRatio) - } - if m.SymbolHitRatio != 1.0 { - t.Errorf("SymbolHitRatio: got %.3f, want 1.0", m.SymbolHitRatio) - } - if m.StrictSymbolHitRatio != 1.0 { - t.Errorf("StrictSymbolHitRatio: got %.3f, want 1.0", m.StrictSymbolHitRatio) - } - if m.TentativeSymbolHitRatio != 0.0 { - t.Errorf("TentativeSymbolHitRatio: got %.3f, want 0.0", m.TentativeSymbolHitRatio) - } - if m.ToolAwareStrictRatio != 1.0 { - t.Errorf("ToolAwareStrictRatio: got %.3f, want 1.0", m.ToolAwareStrictRatio) - } - if m.ToolAwareTentativeRatio != 1.0 { - t.Errorf("ToolAwareTentativeRatio: got %.3f, want 1.0", m.ToolAwareTentativeRatio) - } - if m.LLMStrictBias != 1.0 { - t.Errorf("LLMStrictBias: got %.3f, want 1.0", m.LLMStrictBias) - } - if m.StrictContaminationRate != 0.0 { - t.Errorf("StrictContaminationRate: got %.3f, want 0.0", m.StrictContaminationRate) - } - if m.TotalToolCalls != 3 { - t.Errorf("TotalToolCalls: got %d, want 3", m.TotalToolCalls) - } - if m.CcgToolCalls != 2 { - t.Errorf("CcgToolCalls: got %d, want 2", m.CcgToolCalls) - } - if m.TotalInputTokens != 200 { - t.Errorf("TotalInputTokens: got %d, want 200", m.TotalInputTokens) - } -} - -func TestAnalyzeRun_MultipleResults(t *testing.T) { - run := &benchmark.BenchmarkRun{ - Mode: "with-ccg", - Results: []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}, Answer: "Foo"}, - {QueryID: "q2", FilesRead: []string{"b.go"}, Answer: "Bar"}, - }, - } - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go"}, ExpectedSymbols: []string{"Foo"}}, - {ID: "q2", ExpectedFiles: []string{"b.go"}, ExpectedSymbols: []string{"Bar"}}, - }, - } - matches := benchmark.AnalyzeRun(run, corpus) - if len(matches) != 2 { - t.Errorf("expected 2 matches, got %d", len(matches)) - } -} diff --git a/internal/benchmark/compare.go b/internal/benchmark/compare.go deleted file mode 100644 index 931e05b..0000000 --- a/internal/benchmark/compare.go +++ /dev/null @@ -1,17 +0,0 @@ -// @index Comparison helpers for benchmark runs with and without CCG. -package benchmark - -// Compare builds a ComparisonReport from a with-ccg run and an optional without-ccg run. -// Matches contains with-ccg metrics; MatchesWithout contains without-ccg metrics when provided. -// @intent produce one report that captures both benchmark modes and their scored query matches. -func Compare(withCCG *BenchmarkRun, withoutCCG *BenchmarkRun, corpus *Corpus) *ComparisonReport { - report := &ComparisonReport{ - WithCCG: withCCG, - WithoutCCG: withoutCCG, - Matches: AnalyzeRun(withCCG, corpus), - } - if withoutCCG != nil { - report.MatchesWithout = AnalyzeRun(withoutCCG, corpus) - } - return report -} diff --git a/internal/benchmark/compare_test.go b/internal/benchmark/compare_test.go deleted file mode 100644 index b2a6491..0000000 --- a/internal/benchmark/compare_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package benchmark_test - -import ( - "testing" - "time" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func makeRun(mode string, results []benchmark.RunResult) *benchmark.BenchmarkRun { - return &benchmark.BenchmarkRun{Mode: mode, RunAt: time.Now(), Results: results} -} - -func TestCompare_BothRuns(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go"}, ExpectedSymbols: []string{"Foo"}}, - }, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}, Answer: "Foo is here", InputTokens: 100}, - }) - withoutCCG := makeRun("without-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{}, Answer: "not sure", InputTokens: 200}, - }) - report := benchmark.Compare(withCCG, withoutCCG, corpus) - if report.WithCCG == nil { - t.Error("WithCCG should not be nil") - } - if report.WithoutCCG == nil { - t.Error("WithoutCCG should not be nil") - } - if len(report.Matches) == 0 { - t.Error("expected at least one match result") - } -} - -func TestCompare_OnlyWithCCG(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "test"}, - }, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", Answer: "answer"}, - }) - report := benchmark.Compare(withCCG, nil, corpus) - if report.WithCCG == nil { - t.Error("WithCCG should not be nil") - } - if report.WithoutCCG != nil { - t.Errorf("WithoutCCG should be nil, got %+v", report.WithoutCCG) - } -} - -func TestCompare_TokenDiff(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{{ID: "q1"}}, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", InputTokens: 50}, - }) - withoutCCG := makeRun("without-ccg", []benchmark.RunResult{ - {QueryID: "q1", InputTokens: 300}, - }) - report := benchmark.Compare(withCCG, withoutCCG, corpus) - if len(report.Matches) == 0 { - t.Fatal("no matches") - } - if report.Matches[0].TotalInputTokens != 50 { - t.Errorf("Matches should reflect with-ccg tokens (50), got %d", report.Matches[0].TotalInputTokens) - } -} - -func TestCompare_FileHitImprovement(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go", "b.go"}}, - }, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go", "b.go"}}, - }) - withoutCCG := makeRun("without-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}}, - }) - report := benchmark.Compare(withCCG, withoutCCG, corpus) - if len(report.Matches) == 0 { - t.Fatal("no matches") - } - if report.Matches[0].FileHitRatio != 1.0 { - t.Errorf("with-ccg FileHitRatio should be 1.0, got %.3f", report.Matches[0].FileHitRatio) - } -} diff --git a/internal/benchmark/corpus.go b/internal/benchmark/corpus.go deleted file mode 100644 index f113a23..0000000 --- a/internal/benchmark/corpus.go +++ /dev/null @@ -1,71 +0,0 @@ -// @index Benchmark corpus loading, saving, and validation utilities. -package benchmark - -import ( - "fmt" - "os" - "regexp" - "strings" - - "go.yaml.in/yaml/v3" -) - -var validQueryID = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) - -// LoadCorpus reads a queries.yaml file and validates its contents. -// @intent load a benchmark corpus from disk while enforcing schema and ID constraints up front. -func LoadCorpus(path string) (*Corpus, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("read corpus %q: %w", path, err) - } - var c Corpus - if err := yaml.Unmarshal(data, &c); err != nil { - return nil, fmt.Errorf("parse corpus %q: %w", path, err) - } - if err := ValidateCorpus(&c); err != nil { - return nil, err - } - return &c, nil -} - -// SaveCorpus writes a Corpus to a YAML file. -// @intent persist benchmark corpus definitions in the YAML format used by the CLI. -func SaveCorpus(path string, c *Corpus) error { - data, err := yaml.Marshal(c) - if err != nil { - return fmt.Errorf("marshal corpus: %w", err) - } - if err := os.WriteFile(path, data, 0o644); err != nil { - return fmt.Errorf("write corpus %q: %w", path, err) - } - return nil -} - -// ValidateCorpus checks that all queries have required fields and no duplicate IDs. -// @intent reject malformed benchmark corpora before any benchmark execution depends on them. -func ValidateCorpus(c *Corpus) error { - if len(c.Queries) == 0 { - return fmt.Errorf("corpus has no queries") - } - seen := make(map[string]bool, len(c.Queries)) - for i, q := range c.Queries { - if q.ID == "" { - return fmt.Errorf("query[%d] missing id", i) - } - if !validQueryID.MatchString(q.ID) { - return fmt.Errorf("query[%d] id %q contains invalid characters (use only [A-Za-z0-9_.-])", i, q.ID) - } - if q.Description == "" { - return fmt.Errorf("query %q missing description", q.ID) - } - if seen[q.ID] { - return fmt.Errorf("duplicate query id %q", q.ID) - } - seen[q.ID] = true - if strings.Contains(q.Description, markerStart) || strings.Contains(q.Description, markerEnd) { - return fmt.Errorf("query %q description contains reserved benchmark marker", q.ID) - } - } - return nil -} diff --git a/internal/benchmark/corpus_test.go b/internal/benchmark/corpus_test.go deleted file mode 100644 index 4c30708..0000000 --- a/internal/benchmark/corpus_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package benchmark_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestLoadCorpus_ValidYAML(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `version: "1" -queries: - - id: q1 - description: "결제 실패 복구 로직" - expected_files: - - internal/webhook/handler.go - expected_symbols: - - WebhookHandler - difficulty: medium -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - corpus, err := benchmark.LoadCorpus(path) - if err != nil { - t.Fatalf("LoadCorpus: %v", err) - } - if len(corpus.Queries) != 1 { - t.Errorf("Queries count: got %d, want 1", len(corpus.Queries)) - } - if corpus.Queries[0].ID != "q1" { - t.Errorf("ID: got %q, want q1", corpus.Queries[0].ID) - } -} - -func TestLoadCorpus_FileNotFound(t *testing.T) { - _, err := benchmark.LoadCorpus("/nonexistent/queries.yaml") - if err == nil { - t.Error("expected error for missing file") - } -} - -func TestLoadCorpus_MissingID(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - description: "no id here" -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - _, err := benchmark.LoadCorpus(path) - if err == nil { - t.Error("expected validation error for missing id") - } -} - -func TestLoadCorpus_MissingDescription(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - _, err := benchmark.LoadCorpus(path) - if err == nil { - t.Error("expected validation error for missing description") - } -} - -func TestLoadCorpus_DuplicateID(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 - description: "first" - - id: q1 - description: "duplicate" -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - _, err := benchmark.LoadCorpus(path) - if err == nil { - t.Error("expected validation error for duplicate id") - } -} - -func TestSaveCorpus_WritesYAML(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - corpus := &benchmark.Corpus{ - Version: "1", - Queries: []benchmark.Query{ - {ID: "q1", Description: "테스트 쿼리", Difficulty: "easy"}, - }, - } - if err := benchmark.SaveCorpus(path, corpus); err != nil { - t.Fatalf("SaveCorpus: %v", err) - } - loaded, err := benchmark.LoadCorpus(path) - if err != nil { - t.Fatalf("LoadCorpus after save: %v", err) - } - if len(loaded.Queries) != 1 || loaded.Queries[0].ID != "q1" { - t.Errorf("round-trip failed: got %+v", loaded) - } -} - -func TestValidateCorpus_ValidQuery(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "valid query"}, - }, - } - if err := benchmark.ValidateCorpus(corpus); err != nil { - t.Errorf("expected nil error for valid corpus, got: %v", err) - } -} - -func TestValidateCorpus_EmptyCorpus(t *testing.T) { - corpus := &benchmark.Corpus{Queries: []benchmark.Query{}} - if err := benchmark.ValidateCorpus(corpus); err == nil { - t.Error("expected error for empty corpus") - } -} diff --git a/internal/benchmark/jsonl.go b/internal/benchmark/jsonl.go deleted file mode 100644 index c08455f..0000000 --- a/internal/benchmark/jsonl.go +++ /dev/null @@ -1,284 +0,0 @@ -// @index JSONL session parsing and extraction helpers for benchmark analysis. -package benchmark - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "regexp" - "strings" -) - -var ( - reMarkerStart = regexp.MustCompile(`^===BENCHMARK_QUERY_START id=([A-Za-z0-9_.-]+)===$`) - reMarkerEnd = regexp.MustCompile(`^===BENCHMARK_QUERY_END id=([A-Za-z0-9_.-]+)===$`) -) - -// SessionMessage represents one line from a Claude Code session JSONL file. -// @intent model the subset of Claude session JSONL needed to reconstruct benchmark runs. -type SessionMessage struct { - Type string `json:"type"` - Message *MessagePayload `json:"message,omitempty"` - Content json.RawMessage `json:"content,omitempty"` - ToolUseID string `json:"tool_use_id,omitempty"` - IsError bool `json:"is_error,omitempty"` -} - -// MessagePayload is the nested message object inside a SessionMessage. -// @intent expose assistant role, content blocks, and usage data from session lines. -type MessagePayload struct { - Role string `json:"role"` - Content []ContentBlock `json:"content"` - Usage *UsageInfo `json:"usage,omitempty"` -} - -// ContentBlock is a single content item (text, tool_use, etc.). -// @intent represent individual Claude message blocks so benchmark analyzers can inspect tool calls and text. -type ContentBlock struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - Name string `json:"name,omitempty"` // tool_use - ID string `json:"id,omitempty"` - Input json.RawMessage `json:"input,omitempty"` -} - -// UsageInfo holds token usage from Claude's response. -// @intent capture token accounting for one Claude message so benchmark runs can aggregate usage. -type UsageInfo struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -// QuerySegment groups the messages belonging to a single benchmark query. -// @intent isolate the portion of a session that belongs to one marked benchmark query. -type QuerySegment struct { - QueryID string - Messages []SessionMessage -} - -// ParseJSONL reads a Claude Code session JSONL file and returns all parsed lines. -// Lines that are not valid JSON are silently skipped. -// @intent ingest recorded Claude sessions even when they contain partial or non-JSON noise lines. -func ParseJSONL(path string) ([]SessionMessage, error) { - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("open %q: %w", path, err) - } - defer f.Close() - - var msgs []SessionMessage - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 10<<20), 10<<20) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - var m SessionMessage - if err := json.Unmarshal([]byte(line), &m); err != nil { - continue // skip invalid lines - } - msgs = append(msgs, m) - } - return msgs, scanner.Err() -} - -const ( - markerStart = "===BENCHMARK_QUERY_START id=" - markerEnd = "===BENCHMARK_QUERY_END id=" -) - -// extractMarker checks if a SessionMessage contains a benchmark marker. -// Returns (queryID, isStart, found). Uses strict regex to prevent injection via crafted IDs. -// @intent locate benchmark query boundaries inside session transcripts safely. -func extractMarker(m SessionMessage) (queryID string, isStart bool, found bool) { - var text string - if m.Type == "tool_result" { - text = parseFirstText(m.Content) - } else if m.Message != nil { - for _, b := range m.Message.Content { - if b.Type == "text" { - text = b.Text - break - } - } - } - text = strings.TrimSpace(text) - if sub := reMarkerStart.FindStringSubmatch(text); sub != nil { - return sub[1], true, true - } - if sub := reMarkerEnd.FindStringSubmatch(text); sub != nil { - return sub[1], false, true - } - return "", false, false -} - -// ExtractQuerySegments splits a session's messages into per-query segments using markers. -// An unclosed segment (START without END) is included, spanning to the last message. -// @intent reconstruct per-query transcript slices from a whole benchmark session log. -func ExtractQuerySegments(msgs []SessionMessage) ([]QuerySegment, error) { - var segs []QuerySegment - var current *QuerySegment - - for _, m := range msgs { - id, isStart, found := extractMarker(m) - if found { - if isStart { - current = &QuerySegment{QueryID: id} - } else if current != nil && current.QueryID == id { - segs = append(segs, *current) - current = nil - } - continue - } - if current != nil { - current.Messages = append(current.Messages, m) - } - } - // Include unclosed segment - if current != nil { - segs = append(segs, *current) - } - return segs, nil -} - -// ExtractToolCalls collects all tool_use blocks from a segment's messages. -// @intent recover structured tool invocation history for one benchmark query. -func ExtractToolCalls(seg QuerySegment) []ToolCall { - outputs := extractToolOutputs(seg.Messages) - var calls []ToolCall - for _, m := range seg.Messages { - if m.Message == nil { - continue - } - for _, b := range m.Message.Content { - if b.Type == "tool_use" { - var inputStr string - if len(b.Input) > 0 { - inputStr = string(b.Input) - } - call := ToolCall{ - Tool: b.Name, - Input: inputStr, - } - if b.ID != "" { - call.ToolUseID = b.ID - call.Output = outputs[b.ID] - } - calls = append(calls, call) - } - } - } - return calls -} - -// ExtractFilesRead returns deduplicated file paths from Read tool calls in the segment. -// @intent estimate file-inspection behavior from tool-use records without double-counting reads. -func ExtractFilesRead(seg QuerySegment) []string { - var files []string - seen := make(map[string]bool) - for _, m := range seg.Messages { - if m.Message == nil { - continue - } - for _, b := range m.Message.Content { - if b.Type != "tool_use" || b.Name != "Read" { - continue - } - var inp struct { - FilePath string `json:"file_path"` - } - if err := json.Unmarshal(b.Input, &inp); err == nil && inp.FilePath != "" && !seen[inp.FilePath] { - seen[inp.FilePath] = true - files = append(files, inp.FilePath) - } - } - } - return files -} - -// @intent map tool_use ids to their first text payload so later benchmark analysis can inspect tool outputs by invocation. -func extractToolOutputs(messages []SessionMessage) map[string]string { - outputs := make(map[string]string) - for _, m := range messages { - if m.Type != "tool_result" || m.ToolUseID == "" { - continue - } - outputs[m.ToolUseID] = parseFirstText(m.Content) - } - return outputs -} - -// @intent normalize heterogeneous tool result payloads into one comparable trimmed text string. -func parseFirstText(raw json.RawMessage) string { - if len(raw) == 0 { - return "" -} - - var text string - if err := json.Unmarshal(raw, &text); err == nil { - return strings.TrimSpace(text) - } - - var blocks []ContentBlock - if err := json.Unmarshal(raw, &blocks); err == nil { - for _, b := range blocks { - if b.Type == "text" && b.Text != "" { - return strings.TrimSpace(b.Text) - } - } - return "" - } - - var block ContentBlock - if err := json.Unmarshal(raw, &block); err == nil { - if block.Type == "text" { - return strings.TrimSpace(block.Text) - } - } - return "" -} - -// ExtractTokens sums input and output token counts across all messages in a segment. -// @intent aggregate token usage for the full lifecycle of one benchmark query. -func ExtractTokens(seg QuerySegment) (inputTokens, outputTokens int) { - for _, m := range seg.Messages { - if m.Message != nil && m.Message.Usage != nil { - inputTokens += m.Message.Usage.InputTokens - outputTokens += m.Message.Usage.OutputTokens - } - } - return -} - -// ExtractAnswer returns the text of the last assistant text block in the segment. -// @intent capture the final assistant answer that should be evaluated for symbol hits. -func ExtractAnswer(seg QuerySegment) string { - var last string - for _, m := range seg.Messages { - if m.Message == nil || m.Message.Role != "assistant" { - continue - } - for _, b := range m.Message.Content { - if b.Type == "text" && b.Text != "" { - last = b.Text - } - } - } - return last -} - -// ExtractRunResult builds a RunResult from a query segment. -// @intent convert an extracted query segment into the canonical benchmark run result shape. -func ExtractRunResult(queryID string, seg QuerySegment) RunResult { - in, out := ExtractTokens(seg) - return RunResult{ - QueryID: queryID, - ToolCalls: ExtractToolCalls(seg), - FilesRead: ExtractFilesRead(seg), - Answer: ExtractAnswer(seg), - InputTokens: in, - OutputTokens: out, - } -} diff --git a/internal/benchmark/jsonl_test.go b/internal/benchmark/jsonl_test.go deleted file mode 100644 index 7d09ea1..0000000 --- a/internal/benchmark/jsonl_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package benchmark_test - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -// sampleJSONL returns lines representing a minimal Claude Code session JSONL. -// It includes START/END markers inside tool_result content. -func writeSampleJSONL(t *testing.T, lines []string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "session.jsonl") - var content string - for _, l := range lines { - content += l + "\n" - } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - return path -} - -func TestParseJSONL_ValidSession(t *testing.T) { - path := writeSampleJSONL(t, []string{ - `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"world"}],"usage":{"input_tokens":10,"output_tokens":5}}}`, - }) - msgs, err := benchmark.ParseJSONL(path) - if err != nil { - t.Fatalf("ParseJSONL: %v", err) - } - if len(msgs) != 2 { - t.Errorf("got %d messages, want 2", len(msgs)) - } -} - -func TestParseJSONL_EmptyFile(t *testing.T) { - path := writeSampleJSONL(t, nil) - msgs, err := benchmark.ParseJSONL(path) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(msgs) != 0 { - t.Errorf("expected 0 messages, got %d", len(msgs)) - } -} - -func TestParseJSONL_InvalidLine(t *testing.T) { - path := writeSampleJSONL(t, []string{ - `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]}}`, - `NOT VALID JSON {{{`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"ok"}]}}`, - }) - msgs, err := benchmark.ParseJSONL(path) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(msgs) != 2 { - t.Errorf("expected 2 valid messages (invalid skipped), got %d", len(msgs)) - } -} - -// markerLine produces a tool_result line containing a benchmark marker. -func markerLine(markerText string) string { - return `{"type":"tool_result","tool_use_id":"x","content":"` + markerText + `"}` -} - -func TestExtractQuerySegments_TwoMarkers(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"answer1"}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - markerLine(`===BENCHMARK_QUERY_START id=q2===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"answer2"}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q2===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - t.Fatalf("ExtractQuerySegments: %v", err) - } - if len(segs) != 2 { - t.Errorf("expected 2 segments, got %d", len(segs)) - } - if segs[0].QueryID != "q1" { - t.Errorf("seg[0].QueryID: got %q, want q1", segs[0].QueryID) - } - if segs[1].QueryID != "q2" { - t.Errorf("seg[1].QueryID: got %q, want q2", segs[1].QueryID) - } -} - -func TestExtractQuerySegments_NoMarkers(t *testing.T) { - path := writeSampleJSONL(t, []string{ - `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`, - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - t.Fatalf("ExtractQuerySegments: %v", err) - } - if len(segs) != 0 { - t.Errorf("expected 0 segments, got %d", len(segs)) - } -} - -func TestExtractQuerySegments_UnclosedMarker(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"partial answer"}]}}`, - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - t.Fatalf("ExtractQuerySegments: %v", err) - } - if len(segs) != 1 { - t.Errorf("expected 1 open segment, got %d", len(segs)) - } - if segs[0].QueryID != "q1" { - t.Errorf("seg.QueryID: got %q, want q1", segs[0].QueryID) - } -} - -func TestExtractToolCalls_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"mcp__ccg__search","input":{"query":"retry"}}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t2","name":"Read","input":{"file_path":"internal/foo.go"}}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - if len(segs) == 0 { - t.Fatal("no segments") - } - calls := benchmark.ExtractToolCalls(segs[0]) - if len(calls) != 2 { - t.Errorf("expected 2 tool calls, got %d", len(calls)) - } - if calls[0].Tool != "mcp__ccg__search" { - t.Errorf("calls[0].Tool: got %q", calls[0].Tool) - } -} - -func TestExtractToolCalls_WithToolResultOutput(t *testing.T) { - t.Helper() - toolResult := `{"pattern":"callers_of","results":[{"qualified_name":"Foo","confidence":"strict"}]}` - toolResultJSON, err := json.Marshal(toolResult) - if err != nil { - t.Fatalf("json.Marshal: %v", err) - } - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"x1","name":"mcp__ccg__query_graph","input":{"pattern":"callers_of","target":"pkg.Foo"}}]}}`, - `{"type":"tool_result","tool_use_id":"x1","content":` + string(toolResultJSON) + `}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - calls := benchmark.ExtractToolCalls(segs[0]) - if len(calls) != 1 { - t.Fatalf("expected 1 tool call, got %d", len(calls)) - } - if calls[0].ToolUseID != "x1" { - t.Errorf("ToolUseID: got %q", calls[0].ToolUseID) - } - if calls[0].Output != toolResult { - t.Errorf("Output: got %q want %q", calls[0].Output, toolResult) - } -} - -func TestExtractToolCalls_WithToolResultArrayContent(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"x1","name":"mcp__ccg__query_graph","input":{"pattern":"callees_of","target":"pkg.Bar"}}]}}`, - `{"type":"tool_result","tool_use_id":"x1","content":[{"type":"text","text":"{\"pattern\":\"callees_of\",\"results\":[{\"qualified_name\":\"Bar\",\"confidence\":\"strict\"}]}"}]}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - calls := benchmark.ExtractToolCalls(segs[0]) - if len(calls) != 1 { - t.Fatalf("expected 1 tool call, got %d", len(calls)) - } - if calls[0].Output == "" { - t.Fatalf("tool output should be captured") - } - if !strings.Contains(calls[0].Output, `"qualified_name":"Bar"`) { - t.Errorf("tool output not captured as JSON text: %q", calls[0].Output) - } -} - -func TestExtractFilesRead_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"Read","input":{"file_path":"internal/webhook/handler.go"}}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t2","name":"Glob","input":{"pattern":"internal/**/*.go"}}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - files := benchmark.ExtractFilesRead(segs[0]) - if len(files) != 1 { - t.Errorf("expected 1 file (only Read, not Glob), got %d: %v", len(files), files) - } - if files[0] != "internal/webhook/handler.go" { - t.Errorf("files[0]: got %q", files[0]) - } -} - -func TestExtractTokens_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"first"}],"usage":{"input_tokens":100,"output_tokens":20}}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"second"}],"usage":{"input_tokens":50,"output_tokens":10}}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - in, out := benchmark.ExtractTokens(segs[0]) - if in != 150 { - t.Errorf("InputTokens: got %d, want 150", in) - } - if out != 30 { - t.Errorf("OutputTokens: got %d, want 30", out) - } -} - -func TestExtractAnswer_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"first message"}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"final answer"}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - answer := benchmark.ExtractAnswer(segs[0]) - if answer != "final answer" { - t.Errorf("answer: got %q, want %q", answer, "final answer") - } -} diff --git a/internal/benchmark/report.go b/internal/benchmark/report.go deleted file mode 100644 index 2540da2..0000000 --- a/internal/benchmark/report.go +++ /dev/null @@ -1,53 +0,0 @@ -// @index Markdown report generation for benchmark comparisons. -package benchmark - -import ( - "fmt" - "os" - "strings" -) - -// WriteReport renders a ComparisonReport as markdown and writes it to outPath. -// @intent turn benchmark comparison data into a readable artifact for sharing and regression tracking. -func WriteReport(report *ComparisonReport, outPath string) error { - if report.WithCCG == nil { - return fmt.Errorf("comparison report missing with-ccg run") - } - var sb strings.Builder - - sb.WriteString("# CCG Benchmark Report\n\n") - - // Summary section - sb.WriteString("## Summary\n\n") - fmt.Fprintf(&sb, "- Mode: `%s`\n", report.WithCCG.Mode) - fmt.Fprintf(&sb, "- Queries: %d\n", len(report.WithCCG.Results)) - if report.WithoutCCG != nil { - fmt.Fprintf(&sb, "- Comparison mode: `%s` vs `%s`\n", report.WithCCG.Mode, report.WithoutCCG.Mode) - } - sb.WriteString("\n") - - // Per-query results table - sb.WriteString("## Query Results\n\n") - sb.WriteString("| Query ID | File Hit | Symbol Hit | Tool Calls | CCG Calls | Input Tokens |\n") - sb.WriteString("|----------|----------|------------|------------|-----------|-------------|\n") - for _, m := range report.Matches { - fmt.Fprintf(&sb, "| %s | %.2f | %.2f | %d | %d | %d |\n", - m.QueryID, m.FileHitRatio, m.SymbolHitRatio, - m.TotalToolCalls, m.CcgToolCalls, m.TotalInputTokens) - } - sb.WriteString("\n") - - if len(report.MatchesWithout) > 0 { - sb.WriteString("## Without CCG Results\n\n") - sb.WriteString("| Query ID | File Hit | Symbol Hit | Tool Calls | CCG Calls | Input Tokens |\n") - sb.WriteString("|----------|----------|------------|------------|-----------|-------------|\n") - for _, m := range report.MatchesWithout { - fmt.Fprintf(&sb, "| %s | %.2f | %.2f | %d | %d | %d |\n", - m.QueryID, m.FileHitRatio, m.SymbolHitRatio, - m.TotalToolCalls, m.CcgToolCalls, m.TotalInputTokens) - } - sb.WriteString("\n") - } - - return os.WriteFile(outPath, []byte(sb.String()), 0o644) -} diff --git a/internal/benchmark/report_test.go b/internal/benchmark/report_test.go deleted file mode 100644 index 56043c1..0000000 --- a/internal/benchmark/report_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package benchmark_test - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func makeReport(withoutCCG *benchmark.BenchmarkRun) *benchmark.ComparisonReport { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go"}, ExpectedSymbols: []string{"Foo"}}, - {ID: "q2", ExpectedFiles: []string{"b.go"}, ExpectedSymbols: []string{"Bar"}}, - }, - } - withCCG := &benchmark.BenchmarkRun{ - Mode: "with-ccg", - RunAt: time.Now(), - Results: []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}, Answer: "Foo is here", InputTokens: 100, - ToolCalls: []benchmark.ToolCall{{Tool: "mcp__ccg__search"}}}, - {QueryID: "q2", FilesRead: []string{}, Answer: "unknown", InputTokens: 150}, - }, - } - return benchmark.Compare(withCCG, withoutCCG, corpus) -} - -func TestReport_ContainsSummary(t *testing.T) { - report := makeReport(nil) - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport: %v", err) - } - data, _ := os.ReadFile(out) - if !strings.Contains(string(data), "## Summary") { - t.Errorf("report missing '## Summary' section:\n%s", string(data)) - } -} - -func TestReport_ContainsQueryTable(t *testing.T) { - report := makeReport(nil) - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport: %v", err) - } - data, _ := os.ReadFile(out) - content := string(data) - if !strings.Contains(content, "q1") { - t.Error("report missing query ID q1") - } - if !strings.Contains(content, "q2") { - t.Error("report missing query ID q2") - } -} - -func TestReport_NoWithoutCCG(t *testing.T) { - report := makeReport(nil) - if report.WithoutCCG != nil { - t.Fatal("test setup error: WithoutCCG should be nil") - } - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport with single run: %v", err) - } - data, _ := os.ReadFile(out) - if len(data) == 0 { - t.Error("expected non-empty report") - } -} - -func TestReport_WritesToFile(t *testing.T) { - report := makeReport(nil) - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport: %v", err) - } - if _, err := os.Stat(out); os.IsNotExist(err) { - t.Error("report file was not created") - } -} diff --git a/internal/benchmark/runner.go b/internal/benchmark/runner.go deleted file mode 100644 index 44e8373..0000000 --- a/internal/benchmark/runner.go +++ /dev/null @@ -1,181 +0,0 @@ -// @index Benchmark runner orchestration for Claude CLI subprocess execution. -package benchmark - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strconv" - "time" -) - -// Executor abstracts the subprocess execution so tests can inject a mock. -// dir sets the working directory for the subprocess; empty means inherit from parent. -// @intent decouple benchmark orchestration from the concrete Claude CLI process implementation. -type Executor interface { - Execute(ctx context.Context, args []string, prompt, dir string) ([]byte, error) -} - -// RunnerConfig holds configuration for a benchmark run. -// @intent collect execution mode, working directory, and timeout settings for one benchmark run. -type RunnerConfig struct { - Mode string // "with-ccg" | "without-ccg" - CWD string // benchmark working directory - MaxToolCalls int - TimeoutSec int -} - -// DefaultRunnerConfig returns a RunnerConfig with sensible defaults. -// @intent provide conservative benchmark defaults that work for most local runs. -func DefaultRunnerConfig() RunnerConfig { - return RunnerConfig{ - MaxToolCalls: 50, - TimeoutSec: 120, - } -} - -// claudeOutput is the structure of `claude -p --output-format json` output. -// @intent decode the subset of Claude CLI JSON output needed for benchmark scoring. -type claudeOutput struct { - IsError bool `json:"is_error"` - Result string `json:"result"` - Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - } `json:"usage"` -} - -// buildArgs constructs CLI args for a specific run. -// mcpConfigFile is the path to an empty MCP config file for without-ccg mode. -// @intent centralize Claude CLI argument construction for both benchmark modes. -func buildArgs(cfg RunnerConfig, mcpConfigFile string) []string { - args := []string{"-p", "--output-format", "json"} - if cfg.MaxToolCalls > 0 { - args = append(args, "--max-turns", strconv.Itoa(cfg.MaxToolCalls)) - } - if cfg.Mode == "without-ccg" { - args = append(args, "--strict-mcp-config", "--mcp-config", mcpConfigFile) - } - return args -} - -// BuildClaudeArgs constructs the CLI argument slice for `claude -p`. -// Working directory is set on the subprocess directly (not via --cwd flag). -// In "without-ccg" mode, --strict-mcp-config disables all MCP servers. -// NOTE: For actual runs, use Runner.Run which creates a real temp file for --mcp-config. -// @intent expose deterministic argument construction for tests and tooling that inspect benchmark invocation behavior. -func BuildClaudeArgs(cfg RunnerConfig) []string { - if cfg.Mode == "without-ccg" { - return buildArgs(cfg, `{"mcpServers":{}}`) - } - return buildArgs(cfg, "") -} - -// BuildPrompt creates the prompt string that wraps a query with benchmark markers. -// The markers allow the JSONL analyzer to locate query boundaries. -// @intent surround benchmark queries with markers so later session analysis can recover exact query segments. -func BuildPrompt(q Query) string { - return fmt.Sprintf( - "===BENCHMARK_QUERY_START id=%s===\n%s\n===BENCHMARK_QUERY_END id=%s===", - q.ID, q.Description, q.ID, - ) -} - -// Runner executes benchmark queries sequentially using the provided Executor. -// @intent own the end-to-end lifecycle of benchmark query execution for one mode. -type Runner struct { - cfg RunnerConfig - exec Executor -} - -// NewRunner creates a Runner with the given config and executor. -// @intent assemble benchmark orchestration from execution config and a subprocess adapter. -func NewRunner(cfg RunnerConfig, exec Executor) *Runner { - return &Runner{cfg: cfg, exec: exec} -} - -// Run executes each query in the corpus and returns a BenchmarkRun. -// Executor errors are captured in RunResult.Error rather than aborting the run. -// @intent execute the full benchmark corpus while preserving per-query failures in the final report. -func (r *Runner) Run(ctx context.Context, corpus *Corpus) (*BenchmarkRun, error) { - run := &BenchmarkRun{ - Mode: r.cfg.Mode, - RunAt: time.Now(), - Results: make([]RunResult, 0, len(corpus.Queries)), - } - - mcpConfigFile, cleanup, err := r.prepareMCPConfig() - if err != nil { - return nil, err - } - defer cleanup() - - args := buildArgs(r.cfg, mcpConfigFile) - for _, q := range corpus.Queries { - start := time.Now() - prompt := BuildPrompt(q) - var ( - out []byte - execErr error - ) - func() { - qctx := ctx - if r.cfg.TimeoutSec > 0 { - var cancel context.CancelFunc - qctx, cancel = context.WithTimeout(ctx, time.Duration(r.cfg.TimeoutSec)*time.Second) - defer cancel() - } - out, execErr = r.exec.Execute(qctx, args, prompt, r.cfg.CWD) - }() - elapsed := time.Since(start).Milliseconds() - - res := RunResult{ - QueryID: q.ID, - ElapsedMs: elapsed, - } - if execErr != nil { - res.Error = execErr.Error() - } else { - r.parseClaudeOutput(out, &res) - } - run.Results = append(run.Results, res) - } - return run, nil -} - -// prepareMCPConfig creates a temp JSON file for without-ccg mode so that -// --mcp-config receives a file path rather than inline JSON (avoiding the -// variadic flag consuming the prompt from stdin). -// @intent disable MCP cleanly in without-ccg mode without confusing the Claude CLI argument parser. -func (r *Runner) prepareMCPConfig() (path string, cleanup func(), err error) { - if r.cfg.Mode != "without-ccg" { - return "", func() {}, nil - } - f, ferr := os.CreateTemp("", "ccg-bench-*.json") - if ferr != nil { - return "", nil, fmt.Errorf("create mcp config: %w", ferr) - } - if _, werr := f.WriteString(`{"mcpServers":{}}`); werr != nil { - f.Close() - os.Remove(f.Name()) - return "", nil, fmt.Errorf("write mcp config: %w", werr) - } - f.Close() - return f.Name(), func() { os.Remove(f.Name()) }, nil -} - -// parseClaudeOutput extracts token counts and answer text from claude's JSON output. -// @intent translate Claude CLI JSON output into benchmark result fields without failing the whole run on parse misses. -func (r *Runner) parseClaudeOutput(out []byte, res *RunResult) { - if len(out) == 0 { - return - } - var co claudeOutput - if err := json.Unmarshal(out, &co); err != nil { - return - } - res.InputTokens = co.Usage.InputTokens - res.OutputTokens = co.Usage.OutputTokens - res.Answer = co.Result -} diff --git a/internal/benchmark/runner_test.go b/internal/benchmark/runner_test.go deleted file mode 100644 index a4f93ef..0000000 --- a/internal/benchmark/runner_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package benchmark_test - -import ( - "context" - "errors" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestRunnerConfig_Defaults(t *testing.T) { - cfg := benchmark.DefaultRunnerConfig() - if cfg.MaxToolCalls != 50 { - t.Errorf("MaxToolCalls: got %d, want 50", cfg.MaxToolCalls) - } - if cfg.TimeoutSec != 120 { - t.Errorf("TimeoutSec: got %d, want 120", cfg.TimeoutSec) - } -} - -func TestBuildClaudeArgs_WithCWD(t *testing.T) { - // CWD is passed to Executor.Execute as dir, not as a --cwd CLI flag. - cfg := benchmark.RunnerConfig{ - Mode: "with-ccg", - CWD: "/tmp/benchmark-run", - MaxToolCalls: 50, - TimeoutSec: 120, - } - args := benchmark.BuildClaudeArgs(cfg) - for _, a := range args { - if a == "--cwd" { - t.Errorf("--cwd flag must not appear in args (set via cmd.Dir): %v", args) - } - } -} - -func TestBuildClaudeArgs_OutputFormat(t *testing.T) { - cfg := benchmark.DefaultRunnerConfig() - args := benchmark.BuildClaudeArgs(cfg) - found := false - for i, a := range args { - if a == "--output-format" && i+1 < len(args) && args[i+1] == "json" { - found = true - } - } - if !found { - t.Errorf("expected --output-format json in args: %v", args) - } -} - -// mockExecutor implements benchmark.Executor for testing. -type mockExecutor struct { - calls int - err error - output []byte -} - -func (m *mockExecutor) Execute(_ context.Context, _ []string, _, _ string) ([]byte, error) { - m.calls++ - if m.err != nil { - return nil, m.err - } - return m.output, nil -} - -func TestMockRunner_ExecutesEachQuery(t *testing.T) { - exec := &mockExecutor{output: []byte(`{}`)} - cfg := benchmark.DefaultRunnerConfig() - runner := benchmark.NewRunner(cfg, exec) - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "query one"}, - {ID: "q2", Description: "query two"}, - {ID: "q3", Description: "query three"}, - }, - } - run, err := runner.Run(context.Background(), corpus) - if err != nil { - t.Fatalf("Run: %v", err) - } - if len(run.Results) != 3 { - t.Errorf("expected 3 results, got %d", len(run.Results)) - } - if exec.calls != 3 { - t.Errorf("expected 3 executor calls, got %d", exec.calls) - } -} - -func TestMockRunner_HandlesError(t *testing.T) { - exec := &mockExecutor{err: errors.New("claude not found")} - cfg := benchmark.DefaultRunnerConfig() - runner := benchmark.NewRunner(cfg, exec) - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "fail query"}, - }, - } - run, err := runner.Run(context.Background(), corpus) - if err != nil { - t.Fatalf("Run should not return error (errors go into RunResult): %v", err) - } - if len(run.Results) != 1 { - t.Fatalf("expected 1 result, got %d", len(run.Results)) - } - if run.Results[0].Error == "" { - t.Error("expected RunResult.Error to be set on executor error") - } -} - -func TestBuildPromptWithMarkers(t *testing.T) { - q := benchmark.Query{ID: "q1", Description: "결제 실패 복구 로직"} - prompt := benchmark.BuildPrompt(q) - if !strings.Contains(prompt, "===BENCHMARK_QUERY_START id=q1===") { - t.Errorf("prompt missing START marker: %q", prompt) - } - if !strings.Contains(prompt, "===BENCHMARK_QUERY_END id=q1===") { - t.Errorf("prompt missing END marker: %q", prompt) - } - if !strings.Contains(prompt, q.Description) { - t.Errorf("prompt missing description: %q", prompt) - } -} diff --git a/internal/benchmark/schema.go b/internal/benchmark/schema.go deleted file mode 100644 index 0a4fd7e..0000000 --- a/internal/benchmark/schema.go +++ /dev/null @@ -1,90 +0,0 @@ -// Package benchmark provides types and utilities for benchmarking ccg MCP tool effectiveness. -package benchmark - -import "time" - -// Query represents a single benchmark query with expected results. -// @intent define one benchmark prompt and the files or symbols it is expected to surface. -type Query struct { - ID string `yaml:"id" json:"id"` - Description string `yaml:"description" json:"description"` - ExpectedFiles []string `yaml:"expected_files" json:"expected_files,omitempty"` - ExpectedSymbols []string `yaml:"expected_symbols" json:"expected_symbols,omitempty"` - ExpectedStrictSymbols []string `yaml:"expected_strict_symbols" json:"expected_strict_symbols,omitempty"` - ExpectedTentativeSymbols []string `yaml:"expected_tentative_symbols" json:"expected_tentative_symbols,omitempty"` - Difficulty string `yaml:"difficulty" json:"difficulty,omitempty"` -} - -// Corpus holds the collection of benchmark queries. -// @intent group benchmark queries into a reusable corpus that can be run and validated together. -type Corpus struct { - Version string `yaml:"version" json:"version,omitempty"` - Queries []Query `yaml:"queries" json:"queries"` -} - -// ToolCall records a single tool invocation during query execution. -// @intent capture the tool usage footprint of one benchmarked query execution. -type ToolCall struct { - Tool string `json:"tool"` - ToolUseID string `json:"tool_use_id,omitempty"` - Input string `json:"input,omitempty"` - Output string `json:"output,omitempty"` -} - -// RunResult captures the outcome of executing a single query. -// @intent store answer text, tool usage, timing, and token counts for one benchmark query. -type RunResult struct { - QueryID string `json:"query_id"` - ToolCalls []ToolCall `json:"tool_calls,omitempty"` - FilesRead []string `json:"files_read,omitempty"` - Answer string `json:"answer,omitempty"` - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - ElapsedMs int64 `json:"elapsed_ms"` - Error string `json:"error,omitempty"` -} - -// BenchmarkRun holds all results from a single benchmark execution. -// @intent represent one complete benchmark session for a given execution mode. -type BenchmarkRun struct { - Mode string `json:"mode"` - RunAt time.Time `json:"run_at"` - Results []RunResult `json:"results"` -} - -// ResultByID returns the RunResult for the given query ID, or nil if not found. -// @intent support direct lookup of a query result inside a benchmark run. -func (r *BenchmarkRun) ResultByID(id string) *RunResult { - for i := range r.Results { - if r.Results[i].QueryID == id { - return &r.Results[i] - } - } - return nil -} - -// MatchResult holds the computed match metrics for a single query. -// @intent summarize scored file, symbol, tool, and token metrics for one benchmark query. -type MatchResult struct { - QueryID string `json:"query_id"` - FileHitRatio float64 `json:"file_hit_ratio"` - SymbolHitRatio float64 `json:"symbol_hit_ratio"` - StrictSymbolHitRatio float64 `json:"strict_symbol_hit_ratio"` - TentativeSymbolHitRatio float64 `json:"tentative_symbol_hit_ratio"` - LLMStrictBias float64 `json:"llm_strict_bias"` - StrictContaminationRate float64 `json:"strict_contamination_rate"` - ToolAwareStrictRatio float64 `json:"tool_aware_strict_ratio"` - ToolAwareTentativeRatio float64 `json:"tool_aware_tentative_ratio"` - TotalToolCalls int `json:"total_tool_calls"` - CcgToolCalls int `json:"ccg_tool_calls"` - TotalInputTokens int `json:"total_input_tokens"` -} - -// ComparisonReport holds a comparison between two benchmark runs. -// @intent package benchmark runs and their scored matches into one comparison artifact. -type ComparisonReport struct { - WithCCG *BenchmarkRun `json:"with_ccg"` - WithoutCCG *BenchmarkRun `json:"without_ccg,omitempty"` - Matches []MatchResult `json:"matches"` - MatchesWithout []MatchResult `json:"matches_without,omitempty"` -} diff --git a/internal/benchmark/schema_test.go b/internal/benchmark/schema_test.go deleted file mode 100644 index 1f1ba17..0000000 --- a/internal/benchmark/schema_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package benchmark_test - -import ( - "encoding/json" - "testing" - "time" - - "go.yaml.in/yaml/v3" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestQuery_YAMLRoundtrip(t *testing.T) { - q := benchmark.Query{ - ID: "q1", - Description: "결제 실패 복구 로직", - ExpectedFiles: []string{"internal/webhook/handler.go"}, - ExpectedSymbols: []string{"WebhookHandler"}, - Difficulty: "medium", - } - data, err := yaml.Marshal(q) - if err != nil { - t.Fatalf("marshal: %v", err) - } - var got benchmark.Query - if err := yaml.Unmarshal(data, &got); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if got.ID != q.ID { - t.Errorf("ID: got %q, want %q", got.ID, q.ID) - } - if got.Description != q.Description { - t.Errorf("Description: got %q, want %q", got.Description, q.Description) - } - if len(got.ExpectedFiles) != 1 || got.ExpectedFiles[0] != q.ExpectedFiles[0] { - t.Errorf("ExpectedFiles: got %v, want %v", got.ExpectedFiles, q.ExpectedFiles) - } - if got.Difficulty != q.Difficulty { - t.Errorf("Difficulty: got %q, want %q", got.Difficulty, q.Difficulty) - } -} - -func TestRunResult_JSONRoundtrip(t *testing.T) { - r := benchmark.RunResult{ - QueryID: "q1", - ToolCalls: []benchmark.ToolCall{{Tool: "mcp__ccg__search", Input: "query"}}, - FilesRead: []string{"internal/webhook/handler.go"}, - Answer: "WebhookHandler는 ...", - InputTokens: 100, - OutputTokens: 50, - ElapsedMs: 1234, - } - data, err := json.Marshal(r) - if err != nil { - t.Fatalf("marshal: %v", err) - } - var got benchmark.RunResult - if err := json.Unmarshal(data, &got); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if got.QueryID != r.QueryID { - t.Errorf("QueryID: got %q, want %q", got.QueryID, r.QueryID) - } - if len(got.ToolCalls) != 1 || got.ToolCalls[0].Tool != r.ToolCalls[0].Tool { - t.Errorf("ToolCalls: got %v, want %v", got.ToolCalls, r.ToolCalls) - } - if got.InputTokens != r.InputTokens { - t.Errorf("InputTokens: got %d, want %d", got.InputTokens, r.InputTokens) - } - if got.ElapsedMs != r.ElapsedMs { - t.Errorf("ElapsedMs: got %d, want %d", got.ElapsedMs, r.ElapsedMs) - } -} - -func TestBenchmarkRun_ResultByID(t *testing.T) { - run := benchmark.BenchmarkRun{ - Mode: "with-ccg", - RunAt: time.Now(), - Results: []benchmark.RunResult{ - {QueryID: "q1", Answer: "답변1"}, - {QueryID: "q2", Answer: "답변2"}, - }, - } - got := run.ResultByID("q1") - if got == nil { - t.Fatal("expected non-nil result for q1") - } - if got.Answer != "답변1" { - t.Errorf("Answer: got %q, want %q", got.Answer, "답변1") - } -} - -func TestBenchmarkRun_ResultByID_Missing(t *testing.T) { - run := benchmark.BenchmarkRun{ - Mode: "with-ccg", - Results: []benchmark.RunResult{{QueryID: "q1"}}, - } - got := run.ResultByID("not-exist") - if got != nil { - t.Errorf("expected nil for missing ID, got %+v", got) - } -} diff --git a/internal/benchmark/token_bench.go b/internal/benchmark/token_bench.go deleted file mode 100644 index 6c21948..0000000 --- a/internal/benchmark/token_bench.go +++ /dev/null @@ -1,346 +0,0 @@ -// @index Token baseline and graph-assisted retrieval benchmarking helpers. -package benchmark - -import ( - "context" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "gorm.io/gorm" - - "github.com/tae2089/code-context-graph/internal/model" - "github.com/tae2089/code-context-graph/internal/pathutil" -) - -// DefaultNaiveTokensMaxFileBytes caps per-file size considered by the naive baseline. -// @intent prevent oversized files from skewing or dominating the naive token baseline. -const DefaultNaiveTokensMaxFileBytes int64 = 1 << 20 - -// NaiveTokensOptions tunes which files contribute to the naive token baseline. -// @intent control file selection (excludes, size cap) for the naive reading baseline. -type NaiveTokensOptions struct { - Excludes []string - MaxFileBytes int64 -} - -// SearchBackend는 FTS 검색 백엔드 추상화다. -// @intent abstract graph search so token benchmarks can run against any configured backend. -type SearchBackend interface { - Query(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) -} - -// NodeExpander는 노드의 1-hop 이웃과 어노테이션을 가져오는 추상화다. -// nil을 전달하면 확장 없이 기본 검색 결과만 사용한다. -// @intent optionally enrich graph token context with neighbors and annotations beyond raw search hits. -type NodeExpander interface { - GetEdgesFrom(ctx context.Context, nodeID uint) ([]model.Edge, error) - GetNodesByIDs(ctx context.Context, ids []uint) ([]model.Node, error) - GetAnnotation(ctx context.Context, nodeID uint) (*model.Annotation, error) -} - -// EstimateTokens는 텍스트의 대략적인 토큰 수를 반환한다 (4자 = 1토큰). -// @intent provide a cheap, consistent token estimate for baseline-versus-graph comparisons. -func EstimateTokens(text string) int { - return len(text) / 4 -} - -// NaiveTokens는 repoRoot 아래 exts 확장자를 가진 모든 파일의 토큰 수 합계를 반환한다. -// @intent measure the naive full-file reading baseline for a repository slice. -func NaiveTokens(repoRoot string, exts []string) (int, error) { - return NaiveTokensWithOptions(repoRoot, exts, NaiveTokensOptions{}) -} - -// NaiveTokensWithOptions는 repoRoot 아래 exts 확장자를 가진 파일 중 옵션에 맞는 파일만 집계한다. -// @intent compute a configurable naive token baseline while skipping excluded or oversized files. -func NaiveTokensWithOptions(repoRoot string, exts []string, opts NaiveTokensOptions) (int, error) { - extSet := make(map[string]struct{}, len(exts)) - for _, e := range exts { - extSet[e] = struct{}{} - } - maxFileBytes := opts.MaxFileBytes - if maxFileBytes <= 0 { - maxFileBytes = DefaultNaiveTokensMaxFileBytes - } - var total int - err := filepath.WalkDir(repoRoot, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if pathutil.ShouldSkipDir(d.Name()) { - return filepath.SkipDir - } - return nil - } - relPath, relErr := filepath.Rel(repoRoot, path) - if relErr != nil { - return relErr - } - if pathutil.MatchExcludes(opts.Excludes, relPath) { - return nil - } - if _, ok := extSet[filepath.Ext(path)]; !ok { - return nil - } - info, statErr := d.Info() - if statErr != nil { - return nil - } - if info.Size() > maxFileBytes { - return nil - } - data, rerr := os.ReadFile(path) - if rerr != nil { - return nil - } - total += EstimateTokens(string(data)) - return nil - }) - return total, err -} - -// RunTokenBenchOptions bundles optional knobs for the token benchmark execution. -// @intent extend the token benchmark with baseline tuning while keeping the core API stable. -type RunTokenBenchOptions struct { - Naive NaiveTokensOptions -} - -// TokenBenchResult는 단일 쿼리에 대한 토큰 벤치마크 결과다. -// @intent report naive-versus-graph token cost and recall for one benchmark query. -type TokenBenchResult struct { - QueryID string `json:"query_id"` - NaiveTokens int `json:"naive_tokens"` - GraphTokens int `json:"graph_tokens"` - Ratio float64 `json:"ratio"` - SearchElapsedMs int64 `json:"search_elapsed_ms"` - ResultCount int `json:"result_count"` - // Recall: 정답 파일/심볼이 결과에 포함되었는지 측정 - FilesHit int `json:"files_hit"` - FilesTotal int `json:"files_total"` - SymbolsHit int `json:"symbols_hit"` - SymbolsTotal int `json:"symbols_total"` - Recall float64 `json:"recall"` -} - -// extractASCIITerms는 텍스트에서 ASCII 영숫자 단어만 추출한다. -// 한국어 등 비ASCII 문자를 포함한 설명에서 코드 심볼에 해당하는 영어 단어만 반환한다. -// @intent recover code-like search terms from natural-language benchmark descriptions. -func extractASCIITerms(text string) []string { - var terms []string - for _, word := range strings.Fields(text) { - var clean strings.Builder - for _, r := range word { - if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { - clean.WriteRune(r) - } - } - if clean.Len() > 1 { - terms = append(terms, clean.String()) - } - } - return terms -} - -// readLines는 파일에서 startLine~endLine 범위의 텍스트를 반환한다 (1-based). -// @intent capture a node's code snippet when estimating graph-assisted context size. -func readLines(path string, startLine, endLine int) string { - data, err := os.ReadFile(path) - if err != nil { - return "" - } - lines := strings.Split(string(data), "\n") - lo := max(startLine-1, 0) - hi := min(endLine, len(lines)) - if lo >= hi { - return "" - } - return strings.Join(lines[lo:hi], "\n") -} - -// searchAndCollect는 FTS 검색 후 1-hop 확장까지 수행해 노드 목록과 텍스트를 반환한다. -// seen은 호출 간 중복 제거에 사용되며 nil이면 새로 생성한다. -// @intent gather graph-derived context text and nodes for one token benchmark query. -func searchAndCollect( - ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, - query, repoRoot string, limit int, seen map[uint]struct{}, -) (nodes []model.Node, text string, elapsedMs int64, err error) { - if seen == nil { - seen = make(map[uint]struct{}) - } - - terms := extractASCIITerms(query) - if len(terms) == 0 { - return nil, "", 0, nil - } - - // 단어 수에 반비례해 단어당 limit을 조정한다 (총 결과 예산 유지). - limitPerTerm := max(3, limit/len(terms)) - - var sb strings.Builder - writeNode := func(n model.Node) { - nodes = append(nodes, n) - fmt.Fprintf(&sb, "%s %s %s\n", n.QualifiedName, n.Kind, n.FilePath) - if repoRoot != "" && n.FilePath != "" && n.StartLine > 0 { - code := readLines(filepath.Join(repoRoot, n.FilePath), n.StartLine, n.EndLine) - sb.WriteString(code) - sb.WriteByte('\n') - } - } - - // 각 단어를 개별 검색해 FTS5 AND 제약을 피하고 결과를 누적한다. - for _, term := range terms { - start := time.Now() - found, qerr := backend.Query(ctx, db, term, limitPerTerm) - elapsedMs += time.Since(start).Milliseconds() - if qerr != nil { - return nil, "", elapsedMs, qerr - } - for _, n := range found { - if n.ID > 0 { - if _, dup := seen[n.ID]; dup { - continue - } - seen[n.ID] = struct{}{} - } - writeNode(n) - if expander == nil { - continue - } - edges, eerr := expander.GetEdgesFrom(ctx, n.ID) - if eerr != nil { - continue - } - neighborIDs := make([]uint, 0, len(edges)) - for _, e := range edges { - if _, dup := seen[e.ToNodeID]; !dup { - neighborIDs = append(neighborIDs, e.ToNodeID) - } - } - if len(neighborIDs) == 0 { - continue - } - neighbors, nerr := expander.GetNodesByIDs(ctx, neighborIDs) - if nerr != nil { - continue - } - for _, nb := range neighbors { - seen[nb.ID] = struct{}{} - writeNode(nb) - } - if ann, aerr := expander.GetAnnotation(ctx, n.ID); aerr == nil && ann != nil { - fmt.Fprintf(&sb, "annotation: %s\n", ann.RawText) - } - } - } - return nodes, sb.String(), elapsedMs, nil -} - -// GraphTokens는 단일 쿼리로 검색해 토큰 수, 경과 시간(ms), 결과 수를 반환한다. -// @intent measure the token footprint of graph-assisted retrieval for one query. -func GraphTokens(ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, query, repoRoot string, limit int) (tokens int, elapsedMs int64, count int, err error) { - nodes, text, elapsedMs, err := searchAndCollect(ctx, db, backend, expander, query, repoRoot, limit, nil) - if err != nil { - return 0, elapsedMs, 0, err - } - return EstimateTokens(text), elapsedMs, len(nodes), nil -} - -// countFilesHit는 nodes 중 expectedFiles에 해당하는 FilePath를 가진 노드 수를 반환한다. -// @intent measure how many expected files appear in the retrieved graph context. -func countFilesHit(nodes []model.Node, expectedFiles []string) int { - if len(expectedFiles) == 0 { - return 0 - } - found := make(map[string]struct{}, len(nodes)) - for _, n := range nodes { - found[n.FilePath] = struct{}{} - } - hit := 0 - for _, f := range expectedFiles { - if _, ok := found[f]; ok { - hit++ - } - } - return hit -} - -// countSymbolsHit는 nodes 중 QualifiedName에 expectedSymbols 심볼명이 포함된 노드 수를 반환한다. -// @intent measure how many expected symbols appear in the retrieved graph context. -func countSymbolsHit(nodes []model.Node, expectedSymbols []string) int { - if len(expectedSymbols) == 0 { - return 0 - } - hit := 0 - for _, sym := range expectedSymbols { - for _, n := range nodes { - if strings.Contains(n.QualifiedName, sym) { - hit++ - break - } - } - } - return hit -} - -// computeRecall은 파일/심볼 히트율을 0~1로 반환한다. -// @intent collapse file and symbol hit counts into one recall-style coverage signal. -func computeRecall(filesHit, filesTotal, symbolsHit, symbolsTotal int) float64 { - total := filesTotal + symbolsTotal - if total == 0 { - return 0 - } - return float64(filesHit+symbolsHit) / float64(total) -} - -// RunTokenBench는 corpus의 각 쿼리에 대해 naive/graph 토큰과 recall을 비교한다. -// 검색은 항상 Description을 사용하며, expected_symbols/files는 정답 매칭에만 사용한다. -// limit은 쿼리당 총 결과 예산이며 단어 수에 반비례해 단어당 limit이 자동 조정된다. -// @intent execute the full token benchmark corpus and compare naive reading cost against graph retrieval cost. -func RunTokenBench(ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, corpus *Corpus, repoRoot string, exts []string, limit int) ([]TokenBenchResult, error) { - return RunTokenBenchWithOptions(ctx, db, backend, expander, corpus, repoRoot, exts, limit, RunTokenBenchOptions{}) -} - -// @intent extend token benchmarking with baseline tuning options while preserving the core execution flow. -func RunTokenBenchWithOptions(ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, corpus *Corpus, repoRoot string, exts []string, limit int, opts RunTokenBenchOptions) ([]TokenBenchResult, error) { - naive, err := NaiveTokensWithOptions(repoRoot, exts, opts.Naive) - if err != nil { - return nil, err - } - - results := make([]TokenBenchResult, 0, len(corpus.Queries)) - for _, q := range corpus.Queries { - nodes, text, elapsed, qerr := searchAndCollect(ctx, db, backend, expander, q.Description, repoRoot, limit, nil) - if qerr != nil { - return nil, qerr - } - tokens := EstimateTokens(text) - - filesHit := countFilesHit(nodes, q.ExpectedFiles) - symbolsHit := countSymbolsHit(nodes, q.ExpectedSymbols) - filesTotal := len(q.ExpectedFiles) - symbolsTotal := len(q.ExpectedSymbols) - - var ratio float64 - if tokens > 0 { - ratio = float64(naive) / float64(tokens) - } - results = append(results, TokenBenchResult{ - QueryID: q.ID, - NaiveTokens: naive, - GraphTokens: tokens, - Ratio: ratio, - SearchElapsedMs: elapsed, - ResultCount: len(nodes), - FilesHit: filesHit, - FilesTotal: filesTotal, - SymbolsHit: symbolsHit, - SymbolsTotal: symbolsTotal, - Recall: computeRecall(filesHit, filesTotal, symbolsHit, symbolsTotal), - }) - } - return results, nil -} diff --git a/internal/benchmark/token_bench_test.go b/internal/benchmark/token_bench_test.go deleted file mode 100644 index a24c3df..0000000 --- a/internal/benchmark/token_bench_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package benchmark - -import ( - "context" - "os" - "path/filepath" - "testing" - - "gorm.io/gorm" - - "github.com/tae2089/code-context-graph/internal/model" -) - -// mockSearchBackend는 테스트용 검색 백엔드다. -type mockSearchBackend struct { - nodes []model.Node -} - -func (m *mockSearchBackend) Query(_ context.Context, _ *gorm.DB, _ string, _ int) ([]model.Node, error) { - return m.nodes, nil -} - -// mockNodeExpander는 테스트용 노드 확장기다. -type mockNodeExpander struct { - edges map[uint][]model.Edge - nodes map[uint]model.Node -} - -func (m *mockNodeExpander) GetEdgesFrom(_ context.Context, nodeID uint) ([]model.Edge, error) { - return m.edges[nodeID], nil -} - -func (m *mockNodeExpander) GetNodesByIDs(_ context.Context, ids []uint) ([]model.Node, error) { - var result []model.Node - for _, id := range ids { - if n, ok := m.nodes[id]; ok { - result = append(result, n) - } - } - return result, nil -} - -func (m *mockNodeExpander) GetAnnotation(_ context.Context, _ uint) (*model.Annotation, error) { - return nil, nil -} - -func TestEstimateTokens_FourCharsPerToken(t *testing.T) { - if got := EstimateTokens("abcd"); got != 1 { - t.Errorf("want 1, got %d", got) - } - if got := EstimateTokens("abcdefgh"); got != 2 { - t.Errorf("want 2, got %d", got) - } - if got := EstimateTokens(""); got != 0 { - t.Errorf("want 0, got %d", got) - } -} - -func TestNaiveTokens_CountsSourceFiles(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "util.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - // .txt는 집계 제외 - if err := os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("abcdabcdabcd"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokens(dir, []string{".go"}) - if err != nil { - t.Fatal(err) - } - // main.go: 1, util.go: 2 → 합계 3 - if got != 3 { - t.Errorf("want 3, got %d", got) - } -} - -func TestNaiveTokens_EmptyDir(t *testing.T) { - dir := t.TempDir() - got, err := NaiveTokens(dir, []string{".go"}) - if err != nil { - t.Fatal(err) - } - if got != 0 { - t.Errorf("want 0, got %d", got) - } -} - -func TestNaiveTokensWithOptions_SkipsDefaultDirs(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "vendor"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "vendor", "skip.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokensWithOptions(dir, []string{".go"}, NaiveTokensOptions{}) - if err != nil { - t.Fatal(err) - } - if got != 1 { - t.Fatalf("want only main.go counted, got %d", got) - } -} - -func TestNaiveTokensWithOptions_RespectsExcludes(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "gen"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "gen", "skip.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokensWithOptions(dir, []string{".go"}, NaiveTokensOptions{Excludes: []string{"gen"}}) - if err != nil { - t.Fatal(err) - } - if got != 1 { - t.Fatalf("want only main.go counted after exclude, got %d", got) - } -} - -func TestNaiveTokensWithOptions_SkipsLargeFiles(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "small.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "large.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokensWithOptions(dir, []string{".go"}, NaiveTokensOptions{MaxFileBytes: 4}) - if err != nil { - t.Fatal(err) - } - if got != 1 { - t.Fatalf("want only small.go counted under size cap, got %d", got) - } -} - -func TestGraphTokens_SumNodeText(t *testing.T) { - backend := &mockSearchBackend{ - nodes: []model.Node{ - {QualifiedName: "pkg.Foo", Kind: model.NodeKindFunction, FilePath: "foo.go"}, - {QualifiedName: "pkg.Bar", Kind: model.NodeKindFunction, FilePath: "bar.go"}, - }, - } - tokens, elapsed, count, err := GraphTokens(context.Background(), nil, backend, nil, "foo", "", 10) - if err != nil { - t.Fatal(err) - } - if count != 2 { - t.Errorf("want count=2, got %d", count) - } - if tokens <= 0 { - t.Errorf("want tokens>0, got %d", tokens) - } - if elapsed < 0 { - t.Errorf("want elapsed>=0, got %d", elapsed) - } -} - -func TestGraphTokens_WithExpander_IncludesNeighbors(t *testing.T) { - searchBackend := &mockSearchBackend{ - nodes: []model.Node{ - {ID: 1, QualifiedName: "pkg.Foo", Kind: model.NodeKindFunction, FilePath: "foo.go"}, - }, - } - expander := &mockNodeExpander{ - edges: map[uint][]model.Edge{ - 1: {{FromNodeID: 1, ToNodeID: 2, Kind: model.EdgeKindCalls}}, - }, - nodes: map[uint]model.Node{ - 2: {ID: 2, QualifiedName: "pkg.Bar", Kind: model.NodeKindFunction, FilePath: "bar.go"}, - }, - } - - tokensWithout, _, _, err := GraphTokens(context.Background(), nil, searchBackend, nil, "foo", "", 10) - - if err != nil { - t.Fatal(err) - } - tokensWith, _, _, err := GraphTokens(context.Background(), nil, searchBackend, expander, "foo", "", 10) - if err != nil { - t.Fatal(err) - } - if tokensWith <= tokensWithout { - t.Errorf("want tokensWith(%d) > tokensWithout(%d)", tokensWith, tokensWithout) - } -} - -func TestRunTokenBench_RatioCalculated(t *testing.T) { - dir := t.TempDir() - // 400자 = 100 tokens - content := make([]byte, 400) - for i := range content { - content[i] = 'a' - } - if err := os.WriteFile(filepath.Join(dir, "main.go"), content, 0o600); err != nil { - t.Fatal(err) - } - - backend := &mockSearchBackend{ - nodes: []model.Node{ - {QualifiedName: "pkg.Foo", Kind: model.NodeKindFunction, FilePath: "foo.go"}, - }, - } - corpus := &Corpus{ - Queries: []Query{ - {ID: "q1", Description: "foo 함수 설명"}, - }, - } - - results, err := RunTokenBench(context.Background(), nil, backend, nil, corpus, dir, []string{".go"}, 10) - if err != nil { - t.Fatal(err) - } - if len(results) != 1 { - t.Fatalf("want 1 result, got %d", len(results)) - } - r := results[0] - if r.QueryID != "q1" { - t.Errorf("want query_id=q1, got %s", r.QueryID) - } - if r.NaiveTokens != 100 { - t.Errorf("want naive_tokens=100, got %d", r.NaiveTokens) - } - if r.GraphTokens <= 0 { - t.Errorf("want graph_tokens>0, got %d", r.GraphTokens) - } - if r.Ratio <= 0 { - t.Errorf("want ratio>0, got %f", r.Ratio) - } -} diff --git a/internal/cli/benchmark.go b/internal/cli/benchmark.go deleted file mode 100644 index 17dd122..0000000 --- a/internal/cli/benchmark.go +++ /dev/null @@ -1,334 +0,0 @@ -// @index CLI commands for benchmarking CCG-assisted workflows and token savings. -package cli - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/viper" - "github.com/tae2089/trace" - - "github.com/tae2089/code-context-graph/internal/benchmark" - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/store/gormstore" - storesearch "github.com/tae2089/code-context-graph/internal/store/search" -) - -// @intent group benchmark subcommands that measure retrieval quality and token reduction workflows. -func newBenchmarkCmd(deps *Deps) *cobra.Command { - cmd := &cobra.Command{ - Use: "benchmark", - Short: "Benchmark ccg MCP tool effectiveness", - } - cmd.AddCommand( - newBenchmarkInitCmd(), - newBenchmarkValidateCmd(), - newBenchmarkRunCmd(), - newBenchmarkAnalyzeCmd(), - newBenchmarkCompareCmd(), - newBenchmarkReportCmd(), - newBenchmarkTokenBenchCmd(deps), - ) - return cmd -} - -// newBenchmarkInitCmd creates a corpus directory with a template queries.yaml. -// @intent bootstrap a benchmark corpus so users can start with a valid query fixture quickly. -func newBenchmarkInitCmd() *cobra.Command { - var outDir string - cmd := &cobra.Command{ - Use: "init", - Short: "Initialize a benchmark corpus directory with a template queries.yaml", - RunE: func(cmd *cobra.Command, _ []string) error { - if err := os.MkdirAll(outDir, 0o755); err != nil { - return trace.Wrap(err, "create corpus directory") - } - corpus := &benchmark.Corpus{ - Version: "1", - Queries: []benchmark.Query{ - { - ID: "example-01", - Description: "웹훅 push 수신 후 자동 그래프 업데이트 흐름", - ExpectedFiles: []string{"internal/webhook/handler.go"}, - ExpectedSymbols: []string{"WebhookHandler"}, - Difficulty: "medium", - }, - }, - } - path := filepath.Join(outDir, "queries.yaml") - if err := benchmark.SaveCorpus(path, corpus); err != nil { - return trace.Wrap(err, "write template corpus") - } - fmt.Fprintf(stdout(cmd), "Initialized benchmark corpus at %s\n", path) - return nil - }, - } - cmd.Flags().StringVar(&outDir, "out", "testdata/benchmark", "Output directory for corpus files") - return cmd -} - -// newBenchmarkValidateCmd validates a queries.yaml file. -// @intent fail fast on malformed benchmark corpora before expensive benchmark runs start. -func newBenchmarkValidateCmd() *cobra.Command { - var corpusPath string - cmd := &cobra.Command{ - Use: "validate", - Short: "Validate a benchmark corpus YAML file", - RunE: func(cmd *cobra.Command, _ []string) error { - if _, err := benchmark.LoadCorpus(corpusPath); err != nil { - return trace.Wrap(err, "validate corpus") - } - fmt.Fprintf(stdout(cmd), "Corpus %s is valid\n", corpusPath) - return nil - }, - } - cmd.Flags().StringVar(&corpusPath, "corpus", "testdata/benchmark/queries.yaml", "Path to corpus YAML file") - return cmd -} - -// newBenchmarkRunCmd executes each query via `claude -p` subprocess. -// @intent run the benchmark corpus through Claude CLI and capture structured results for later comparison. -func newBenchmarkRunCmd() *cobra.Command { - var corpusPath, cwd, mode, outPath string - cmd := &cobra.Command{ - Use: "run", - Short: "Run benchmark queries via claude CLI subprocess", - RunE: func(cmd *cobra.Command, _ []string) error { - if cwd == "" { - return fmt.Errorf("--cwd is required: set the benchmark working directory") - } - corpus, err := benchmark.LoadCorpus(corpusPath) - if err != nil { - return trace.Wrap(err, "load corpus") - } - cfg := benchmark.DefaultRunnerConfig() - cfg.Mode = mode - cfg.CWD = cwd - runner := benchmark.NewRunner(cfg, &osExecutor{}) - run, err := runner.Run(cmd.Context(), corpus) - if err != nil { - return trace.Wrap(err, "run benchmark") - } - data, err := json.MarshalIndent(run, "", " ") - if err != nil { - return trace.Wrap(err, "marshal run") - } - if outPath != "" { - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return trace.Wrap(err, "write output") - } - fmt.Fprintf(stdout(cmd), "Results saved to %s\n", outPath) - } else { - fmt.Fprintln(stdout(cmd), string(data)) - } - return nil - }, - } - cmd.Flags().StringVar(&corpusPath, "corpus", "testdata/benchmark/queries.yaml", "Path to corpus YAML file") - cmd.Flags().StringVar(&cwd, "cwd", "", "Benchmark working directory (required)") - cmd.Flags().StringVar(&mode, "mode", "with-ccg", "Benchmark mode: with-ccg or without-ccg") - cmd.Flags().StringVar(&outPath, "out", "", "Output file path (JSON); defaults to stdout") - return cmd -} - -// newBenchmarkAnalyzeCmd parses a Claude Code session JSONL and extracts RunResults. -// @intent recover benchmark run results from recorded Claude Code session logs. -func newBenchmarkAnalyzeCmd() *cobra.Command { - var sessionPath string - cmd := &cobra.Command{ - Use: "analyze", - Short: "Parse a Claude Code session JSONL and extract query run results", - RunE: func(cmd *cobra.Command, _ []string) error { - if sessionPath == "" { - return fmt.Errorf("--session is required: path to Claude Code session JSONL file") - } - msgs, err := benchmark.ParseJSONL(sessionPath) - if err != nil { - return trace.Wrap(err, "parse JSONL") - } - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - return trace.Wrap(err, "extract segments") - } - results := make([]benchmark.RunResult, 0, len(segs)) - for _, seg := range segs { - results = append(results, benchmark.ExtractRunResult(seg.QueryID, seg)) - } - data, err := json.MarshalIndent(results, "", " ") - if err != nil { - return trace.Wrap(err, "marshal results") - } - fmt.Fprintln(stdout(cmd), string(data)) - return nil - }, - } - cmd.Flags().StringVar(&sessionPath, "session", "", "Path to Claude Code session JSONL file (required)") - return cmd -} - -// newBenchmarkCompareCmd compares two BenchmarkRun JSON files. -// @intent quantify the delta between with-ccg and without-ccg benchmark runs. -func newBenchmarkCompareCmd() *cobra.Command { - var withPath, withoutPath, corpusPath string - cmd := &cobra.Command{ - Use: "compare", - Short: "Compare two benchmark runs (with-ccg vs without-ccg)", - RunE: func(cmd *cobra.Command, _ []string) error { - withRun, err := loadBenchmarkRun(withPath) - if err != nil { - return trace.Wrap(err, "load with-ccg run") - } - var withoutRun *benchmark.BenchmarkRun - if withoutPath != "" { - withoutRun, err = loadBenchmarkRun(withoutPath) - if err != nil { - return trace.Wrap(err, "load without-ccg run") - } - } - var corpus *benchmark.Corpus - if corpusPath != "" { - corpus, err = benchmark.LoadCorpus(corpusPath) - if err != nil { - return trace.Wrap(err, "load corpus") - } - } else { - corpus = &benchmark.Corpus{} - } - report := benchmark.Compare(withRun, withoutRun, corpus) - data, err := json.MarshalIndent(report, "", " ") - if err != nil { - return trace.Wrap(err, "marshal report") - } - fmt.Fprintln(stdout(cmd), string(data)) - return nil - }, - } - cmd.Flags().StringVar(&withPath, "with", "", "Path to with-ccg run JSON (required)") - cmd.Flags().StringVar(&withoutPath, "without", "", "Path to without-ccg run JSON (optional)") - cmd.Flags().StringVar(&corpusPath, "corpus", "", "Path to corpus YAML for expected matching") - _ = cmd.MarkFlagRequired("with") - return cmd -} - -// newBenchmarkReportCmd generates a markdown report from a ComparisonReport JSON. -// @intent turn raw benchmark comparison output into a shareable markdown summary. -func newBenchmarkReportCmd() *cobra.Command { - var compPath, outPath string - cmd := &cobra.Command{ - Use: "report", - Short: "Generate a markdown report from a comparison JSON file", - RunE: func(cmd *cobra.Command, _ []string) error { - if compPath == "" { - return fmt.Errorf("--comparison is required") - } - data, err := os.ReadFile(compPath) - if err != nil { - return trace.Wrap(err, "read comparison") - } - var report benchmark.ComparisonReport - if err := json.Unmarshal(data, &report); err != nil { - return trace.Wrap(err, "parse comparison") - } - if err := benchmark.WriteReport(&report, outPath); err != nil { - return trace.Wrap(err, "write report") - } - fmt.Fprintf(stdout(cmd), "Report written to %s\n", outPath) - return nil - }, - } - cmd.Flags().StringVar(&compPath, "comparison", "", "Path to comparison JSON file (required)") - cmd.Flags().StringVar(&outPath, "out", "report.md", "Output markdown file path") - return cmd -} - -// loadBenchmarkRun reads and unmarshals a BenchmarkRun from a JSON file. -// @intent centralize benchmark run loading for compare and reporting commands. -func loadBenchmarkRun(path string) (*benchmark.BenchmarkRun, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var run benchmark.BenchmarkRun - if err := json.Unmarshal(data, &run); err != nil { - return nil, err - } - return &run, nil -} - -// newBenchmarkTokenBenchCmd measures token reduction: naive file reading vs CCG graph query. -// @intent measure token savings from CCG-assisted retrieval without involving an LLM response loop. -func newBenchmarkTokenBenchCmd(deps *Deps) *cobra.Command { - var corpusPath, repoRoot, outPath string - var exts []string - var excludePatterns []string - var limit int - cmd := &cobra.Command{ - Use: "token-bench", - Short: "Measure token reduction: naive file reading vs CCG graph query (no LLM)", - RunE: func(cmd *cobra.Command, _ []string) error { - if deps.DB == nil { - return errDBNotInitialized - } - corpus, err := benchmark.LoadCorpus(corpusPath) - if err != nil { - return trace.Wrap(err, "load corpus") - } - backend := storesearch.NewSQLiteBackend() - ctx := ctxns.WithNamespace(cmd.Context(), viper.GetString("namespace")) - results, err := benchmark.RunTokenBenchWithOptions(ctx, deps.DB, backend, gormstore.New(deps.DB), corpus, repoRoot, exts, limit, benchmark.RunTokenBenchOptions{ - Naive: benchmark.NaiveTokensOptions{Excludes: resolveExcludes(excludePatterns)}, - }) - if err != nil { - return trace.Wrap(err, "run token bench") - } - data, err := json.MarshalIndent(results, "", " ") - if err != nil { - return trace.Wrap(err, "marshal results") - } - if outPath != "" { - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return trace.Wrap(err, "write output") - } - fmt.Fprintf(stdout(cmd), "Results saved to %s\n", outPath) - } else { - fmt.Fprintln(stdout(cmd), string(data)) - } - return nil - }, - } - cmd.Flags().StringVar(&corpusPath, "corpus", "testdata/benchmark/queries.yaml", "Path to corpus YAML file") - cmd.Flags().StringVar(&repoRoot, "repo", ".", "Repository root to measure naive tokens") - cmd.Flags().StringVar(&outPath, "out", "", "Output JSON file; defaults to stdout") - cmd.Flags().StringSliceVar(&exts, "exts", []string{".go"}, "Source file extensions to count") - cmd.Flags().StringArrayVar(&excludePatterns, "exclude", nil, "Exclude files/paths from naive token baseline (repeatable)") - cmd.Flags().IntVar(&limit, "limit", 30, "Total result budget per query (auto-split across terms)") - return cmd -} - -// osExecutor implements benchmark.Executor using os/exec to call the `claude` CLI. -// @intent bridge benchmark runner execution to the local Claude CLI process. -type osExecutor struct{} - -// @intent execute one benchmark prompt in a subprocess while preserving CLI-compatible stdin semantics. -func (e *osExecutor) Execute(ctx context.Context, args []string, prompt, dir string) ([]byte, error) { - c := exec.CommandContext(ctx, "claude", args...) - c.Stdin = strings.NewReader(prompt) - if dir != "" { - c.Dir = dir - } - out, err := c.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { - return out, fmt.Errorf("%w: %s", err, exitErr.Stderr) - } - } - return out, err -} diff --git a/internal/cli/benchmark_namespace_test.go b/internal/cli/benchmark_namespace_test.go deleted file mode 100644 index b042fbe..0000000 --- a/internal/cli/benchmark_namespace_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package cli - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/model" -) - -func TestBenchmarkTokenBench_RespectsNamespace(t *testing.T) { - deps, stdout, stderr, db := setupSearchTest(t) - - nodeA := model.Node{Namespace: "ns-a", Name: "SharedA", QualifiedName: "pkg.SharedA", Kind: model.NodeKindFunction, FilePath: "a.go", StartLine: 1, EndLine: 1, Language: "go"} - nodeB := model.Node{Namespace: "ns-b", Name: "SharedB", QualifiedName: "pkg.SharedB", Kind: model.NodeKindFunction, FilePath: "b.go", StartLine: 1, EndLine: 1, Language: "go"} - if err := db.Create(&nodeA).Error; err != nil { - t.Fatalf("create nodeA: %v", err) - } - if err := db.Create(&nodeB).Error; err != nil { - t.Fatalf("create nodeB: %v", err) - } - if err := db.Create(&model.SearchDocument{Namespace: "ns-a", NodeID: nodeA.ID, Content: "sharedterm alpha", Language: "go"}).Error; err != nil { - t.Fatalf("create docA: %v", err) - } - if err := db.Create(&model.SearchDocument{Namespace: "ns-b", NodeID: nodeB.ID, Content: "sharedterm beta", Language: "go"}).Error; err != nil { - t.Fatalf("create docB: %v", err) - } - if err := deps.SearchBackend.Rebuild(ctxns.WithNamespace(context.Background(), "ns-a"), db); err != nil { - t.Fatalf("rebuild ns-a search: %v", err) - } - if err := deps.SearchBackend.Rebuild(ctxns.WithNamespace(context.Background(), "ns-b"), db); err != nil { - t.Fatalf("rebuild ns-b search: %v", err) - } - - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "b.go"), []byte("package p\nfunc SharedB() {}\n"), 0o644); err != nil { - t.Fatalf("write repo file: %v", err) - } - corpusPath := filepath.Join(t.TempDir(), "queries.yaml") - if err := os.WriteFile(corpusPath, []byte(`queries: - - id: q1 - description: sharedterm - expected_symbols: - - SharedB -`), 0o644); err != nil { - t.Fatalf("write corpus: %v", err) - } - - if err := executeCmd(deps, stdout, stderr, "--namespace", "ns-b", "benchmark", "token-bench", "--corpus", corpusPath, "--repo", root, "--exts", ".go"); err != nil { - t.Fatalf("token-bench: %v\nstderr=%s", err, stderr.String()) - } - - var results []benchmark.TokenBenchResult - if err := json.Unmarshal(stdout.Bytes(), &results); err != nil { - t.Fatalf("decode output: %v\nstdout=%s", err, stdout.String()) - } - if len(results) != 1 { - t.Fatalf("results = %d, want 1", len(results)) - } - if results[0].ResultCount == 0 || results[0].SymbolsHit != 1 { - t.Fatalf("token-bench did not use ns-b search results: %+v", results[0]) - } -} diff --git a/internal/cli/benchmark_test.go b/internal/cli/benchmark_test.go deleted file mode 100644 index a3b35a1..0000000 --- a/internal/cli/benchmark_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package cli_test - -import ( - "bytes" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/cli" -) - -// runBenchmarkCmd is a helper that runs `ccg benchmark ` with a fresh Deps. -func runBenchmarkCmd(t *testing.T, args ...string) (stdout string, err error) { - t.Helper() - var buf bytes.Buffer - deps := &cli.Deps{} - root := cli.NewRootCmd(deps) - root.SetOut(&buf) - root.SetErr(&buf) - root.SetArgs(append([]string{"benchmark"}, args...)) - err = root.Execute() - return buf.String(), err -} - -func TestBenchmarkCmd_HasSubcommands(t *testing.T) { - var buf bytes.Buffer - deps := &cli.Deps{} - root := cli.NewRootCmd(deps) - root.SetOut(&buf) - root.SetErr(&buf) - root.SetArgs([]string{"benchmark", "--help"}) - _ = root.Execute() - out := buf.String() - for _, sub := range []string{"init", "validate", "run", "analyze", "compare", "report"} { - if !strings.Contains(out, sub) { - t.Errorf("help output missing subcommand %q:\n%s", sub, out) - } - } -} - -func TestBenchmarkInitCmd_CreatesDir(t *testing.T) { - dir := t.TempDir() - outDir := filepath.Join(dir, "corpus") - _, err := runBenchmarkCmd(t, "init", "--out", outDir) - if err != nil { - t.Fatalf("benchmark init: %v", err) - } - queriesPath := filepath.Join(outDir, "queries.yaml") - if _, err := os.Stat(queriesPath); os.IsNotExist(err) { - t.Errorf("queries.yaml not created at %s", queriesPath) - } -} - -func TestBenchmarkValidateCmd_ValidCorpus(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 - description: "valid query" -` - _ = os.WriteFile(path, []byte(content), 0o644) - _, err := runBenchmarkCmd(t, "validate", "--corpus", path) - if err != nil { - t.Errorf("expected success for valid corpus, got: %v", err) - } -} - -func TestBenchmarkValidateCmd_InvalidCorpus(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 - description: "first" - - id: q1 - description: "duplicate" -` - _ = os.WriteFile(path, []byte(content), 0o644) - _, err := runBenchmarkCmd(t, "validate", "--corpus", path) - if err == nil { - t.Error("expected error for duplicate ID corpus") - } -} - -func TestBenchmarkAnalyzeCmd_MissingSession(t *testing.T) { - _, err := runBenchmarkCmd(t, "analyze") - if err == nil { - t.Error("expected error when --session flag is missing") - } -} - -func TestBenchmarkAnalyzeCmd_ReadsJSONL(t *testing.T) { - dir := t.TempDir() - // Write minimal JSONL with markers - jsonlContent := `{"type":"tool_result","tool_use_id":"x","content":"===BENCHMARK_QUERY_START id=q1==="}` + "\n" + - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"answer here"}],"usage":{"input_tokens":10,"output_tokens":5}}}` + "\n" + - `{"type":"tool_result","tool_use_id":"y","content":"===BENCHMARK_QUERY_END id=q1==="}` + "\n" - sessionPath := filepath.Join(dir, "session.jsonl") - _ = os.WriteFile(sessionPath, []byte(jsonlContent), 0o644) - - out, err := runBenchmarkCmd(t, "analyze", "--session", sessionPath) - if err != nil { - t.Fatalf("benchmark analyze: %v", err) - } - // Output should be valid JSON containing query results - var result interface{} - if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &result); err != nil { - t.Errorf("output is not valid JSON: %v\noutput: %s", err, out) - } -} - -func TestBenchmarkCompareCmd_TwoFiles(t *testing.T) { - dir := t.TempDir() - - writeRun := func(name, mode string) string { - path := filepath.Join(dir, name) - data, _ := json.Marshal(map[string]interface{}{ - "mode": mode, - "run_at": "2026-01-01T00:00:00Z", - "results": []interface{}{}, - }) - _ = os.WriteFile(path, data, 0o644) - return path - } - - withPath := writeRun("with.json", "with-ccg") - withoutPath := writeRun("without.json", "without-ccg") - - out, err := runBenchmarkCmd(t, "compare", "--with", withPath, "--without", withoutPath) - if err != nil { - t.Fatalf("benchmark compare: %v", err) - } - var result interface{} - if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &result); err != nil { - t.Errorf("compare output is not valid JSON: %v\noutput: %s", err, out) - } -} - -func TestBenchmarkReportCmd_GeneratesMarkdown(t *testing.T) { - dir := t.TempDir() - compPath := filepath.Join(dir, "comparison.json") - outPath := filepath.Join(dir, "report.md") - - compData, _ := json.Marshal(map[string]interface{}{ - "with_ccg": map[string]interface{}{ - "mode": "with-ccg", - "run_at": "2026-01-01T00:00:00Z", - "results": []interface{}{}, - }, - "matches": []interface{}{}, - }) - _ = os.WriteFile(compPath, compData, 0o644) - - _, err := runBenchmarkCmd(t, "report", "--comparison", compPath, "--out", outPath) - if err != nil { - t.Fatalf("benchmark report: %v", err) - } - if _, err := os.Stat(outPath); os.IsNotExist(err) { - t.Error("report.md was not created") - } -} - -func TestBenchmarkRunCmd_RequiresCWD(t *testing.T) { - dir := t.TempDir() - corpusPath := filepath.Join(dir, "queries.yaml") - _ = os.WriteFile(corpusPath, []byte("queries:\n - id: q1\n description: test\n"), 0o644) - // Run without --cwd should fail - _, err := runBenchmarkCmd(t, "run", "--corpus", corpusPath) - if err == nil { - t.Error("expected error when --cwd is missing") - } -} diff --git a/internal/cli/eval.go b/internal/cli/eval.go deleted file mode 100644 index e5e23d0..0000000 --- a/internal/cli/eval.go +++ /dev/null @@ -1,102 +0,0 @@ -// @index CLI command for parser and search evaluation against golden corpora. -package cli - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/spf13/cobra" - "github.com/tae2089/trace" - - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/eval" - "github.com/tae2089/code-context-graph/internal/model" - "github.com/tae2089/code-context-graph/internal/parse/treesitter" -) - -// @intent expose parser and search evaluation workflows from the CLI with one shared command surface. -func newEvalCmd(deps *Deps) *cobra.Command { - var corpusDir string - var suite string - var format string - var update bool - - cmd := &cobra.Command{ - Use: "eval", - Short: "Evaluate parser accuracy and search quality against golden corpus", - RunE: func(cmd *cobra.Command, args []string) error { - ns, _ := cmd.Flags().GetString("namespace") - ctx := ctxns.WithNamespace(cmd.Context(), ns) - opts := eval.RunOptions{ - CorpusDir: corpusDir, - Suite: suite, - Format: format, - Update: update, - Walkers: extToLangWalkers(deps.Walkers), - Writer: stdout(cmd), - } - - if (suite == "all" || suite == "search") && deps.DB != nil && deps.SearchBackend != nil { - opts.SearchFn = makeSearchFn(ctx, deps) - } - - _, err := eval.Run(ctx, opts) - if err != nil { - return trace.Wrap(err, "eval") - } - return nil - }, - } - - cmd.Flags().StringVar(&corpusDir, "corpus", "testdata/eval", "Path to golden corpus directory") - cmd.Flags().StringVar(&suite, "suite", "all", "Evaluation suite: all, parser, search") - cmd.Flags().StringVar(&format, "format", "table", "Output format: table, json") - cmd.Flags().BoolVar(&update, "update", false, "Update golden files from current parser output (parser suite only)") - - return cmd -} - -// @intent adapt the configured search backend to the eval package's simple search callback contract. -func makeSearchFn(ctx context.Context, deps *Deps) eval.SearchFunc { - return func(query string, limit int) ([]string, error) { - if deps.DB == nil || deps.SearchBackend == nil { - return nil, fmt.Errorf("database not initialized for search eval") - } - nodes, err := deps.SearchBackend.Query(ctx, deps.DB, query, limit) - if err != nil { - return nil, err - } - return nodeToKeys(nodes), nil - } -} - -// extToLangWalkers converts extension-keyed walkers (e.g. ".go") to language-keyed (e.g. "go"). -// @intent normalize parser walker lookup so eval suites can address languages consistently. -func extToLangWalkers(walkers map[string]*treesitter.Walker) map[string]*treesitter.Walker { - if walkers == nil { - return nil - } - extToLang := map[string]string{ - ".go": "go", ".py": "python", ".ts": "typescript", ".tsx": "typescript", - ".java": "java", ".rb": "ruby", ".js": "javascript", ".jsx": "javascript", - ".c": "c", ".cpp": "cpp", ".rs": "rust", ".kt": "kotlin", ".php": "php", - ".lua": "lua", ".luau": "lua", - } - result := make(map[string]*treesitter.Walker) - for ext, w := range walkers { - if lang, ok := extToLang[ext]; ok { - result[lang] = w - } - } - return result -} - -// @intent convert search result nodes into stable eval comparison keys. -func nodeToKeys(nodes []model.Node) []string { - keys := make([]string, len(nodes)) - for i, n := range nodes { - keys[i] = string(n.Kind) + ":" + n.Name + "@" + filepath.Base(n.FilePath) - } - return keys -} diff --git a/internal/cli/eval_test.go b/internal/cli/eval_test.go deleted file mode 100644 index 77bdb9f..0000000 --- a/internal/cli/eval_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package cli - -import ( - "context" - "testing" - - "gorm.io/gorm" - - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/model" -) - -func TestNodeToKeys_UsesBaseFilePathForEvalMatching(t *testing.T) { - keys := nodeToKeys([]model.Node{{ - Kind: model.NodeKindFunction, - Name: "GetUser", - FilePath: "go/sample.go", - }}) - - if len(keys) != 1 { - t.Fatalf("keys length = %d, want 1", len(keys)) - } - if got, want := keys[0], "function:GetUser@sample.go"; got != want { - t.Fatalf("key = %q, want %q", got, want) - } -} - -type evalSpySearchBackend struct { - queryFn func(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) -} - -func (s *evalSpySearchBackend) Migrate(db *gorm.DB) error { - return nil -} - -func (s *evalSpySearchBackend) Rebuild(ctx context.Context, db *gorm.DB) error { - return nil -} - -func (s *evalSpySearchBackend) RebuildNodes(ctx context.Context, db *gorm.DB, nodeIDs []uint) error { - return nil -} - -func (s *evalSpySearchBackend) PurgeNamespace(ctx context.Context, db *gorm.DB) error { - return nil -} - -func (s *evalSpySearchBackend) Query(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) { - if s.queryFn != nil { - return s.queryFn(ctx, db, query, limit) - } - return nil, nil -} - -func TestMakeSearchFn_UsesProvidedNamespaceContext(t *testing.T) { - deps, _, _ := newTestDeps() - deps.DB = &gorm.DB{} - deps.SearchBackend = &evalSpySearchBackend{queryFn: func(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) { - if got := ctxns.FromContext(ctx); got != "alpha" { - t.Fatalf("query context namespace = %q, want %q", got, "alpha") - } - return []model.Node{{Kind: model.NodeKindFunction, Name: "Foo", FilePath: "foo.go"}}, nil - }} - - searchFn := makeSearchFn(ctxns.WithNamespace(context.Background(), "alpha"), deps) - if _, err := searchFn("foo", 5); err != nil { - t.Fatalf("searchFn returned error: %v", err) - } -} diff --git a/internal/cli/root.go b/internal/cli/root.go index bb31835..948e479 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -161,9 +161,7 @@ func NewRootCmd(deps *Deps) *cobra.Command { newIndexCmd(deps), newLintCmd(deps), newRagIndexCmd(deps), - newEvalCmd(deps), newVersionCmd(deps), - newBenchmarkCmd(deps), ) return rootCmd diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index af95dea..f537ee4 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -8,7 +8,6 @@ import ( "path/filepath" "testing" - "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -225,15 +224,6 @@ func TestRoot_HooksInstallSkipsInitFuncViaParentAnnotation(t *testing.T) { } } -func findSubCmd(root *cobra.Command, name string) *cobra.Command { - for _, c := range root.Commands() { - if c.Name() == name { - return c - } - } - return nil -} - func TestRoot_MissingConfigFileIsIgnored(t *testing.T) { viper.Reset() defer viper.Reset() diff --git a/internal/cli/update_test.go b/internal/cli/update_test.go index 6c5cc1f..e6d1748 100644 --- a/internal/cli/update_test.go +++ b/internal/cli/update_test.go @@ -327,11 +327,3 @@ func Cancelled() {} t.Fatalf("expected context canceled, got %v", err) } } - -func writeGoFileForUpdate(t *testing.T, dir, name, content string) { - t.Helper() - path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } -} diff --git a/internal/eval/metrics.go b/internal/eval/metrics.go deleted file mode 100644 index 57da209..0000000 --- a/internal/eval/metrics.go +++ /dev/null @@ -1,140 +0,0 @@ -// @index Ranking and classification metrics for parser and search evaluation. -package eval - -import "math" - -// ClassificationMetrics holds precision, recall, and F1 counts for one comparison. -// @intent expose set-based scoring fields used by parser node and edge evaluation. -type ClassificationMetrics struct { - TruePositive int - FalsePositive int - FalseNegative int - Precision float64 - Recall float64 - F1 float64 -} - -// ComputeClassification computes set-based precision, recall, and F1 for expected versus actual keys. -// @intent provide one reusable metric primitive for both parser node and edge evaluation. -func ComputeClassification(expected, actual []string) ClassificationMetrics { - expectedSet := make(map[string]bool, len(expected)) - for _, e := range expected { - expectedSet[e] = true - } - actualSet := make(map[string]bool, len(actual)) - for _, a := range actual { - actualSet[a] = true - } - - var m ClassificationMetrics - for a := range actualSet { - if expectedSet[a] { - m.TruePositive++ - } else { - m.FalsePositive++ - } - } - for e := range expectedSet { - if !actualSet[e] { - m.FalseNegative++ - } - } - - if m.TruePositive+m.FalsePositive > 0 { - m.Precision = float64(m.TruePositive) / float64(m.TruePositive+m.FalsePositive) - } - if m.TruePositive+m.FalseNegative > 0 { - m.Recall = float64(m.TruePositive) / float64(m.TruePositive+m.FalseNegative) - } - if m.Precision+m.Recall > 0 { - m.F1 = 2 * m.Precision * m.Recall / (m.Precision + m.Recall) - } - - return m -} - -// @intent measure how many of the top-k ranked results are relevant. -func PrecisionAtK(ranked []string, relevant map[string]bool, k int) float64 { - if k <= 0 || len(ranked) == 0 || len(relevant) == 0 { - return 0 - } - n := k - if n > len(ranked) { - n = len(ranked) - } - hits := 0 - for i := 0; i < n; i++ { - if relevant[ranked[i]] { - hits++ - } - } - return float64(hits) / float64(n) -} - -// @intent measure how much of the relevant set appears within the top-k ranked results. -func RecallAtK(ranked []string, relevant map[string]bool, k int) float64 { - if k <= 0 || len(ranked) == 0 || len(relevant) == 0 { - return 0 - } - n := k - if n > len(ranked) { - n = len(ranked) - } - hits := 0 - for i := 0; i < n; i++ { - if relevant[ranked[i]] { - hits++ - } - } - return float64(hits) / float64(len(relevant)) -} - -// @intent score the position of the first relevant result for ranking evaluation. -func MRR(ranked []string, relevant map[string]bool) float64 { - for i, r := range ranked { - if relevant[r] { - return 1.0 / float64(i+1) - } - } - return 0 -} - -// @intent compare ranked retrieval quality against an ideal ordering using discounted gain. -func NDCG(ranked []string, relevant map[string]bool, k int) float64 { - if k <= 0 || len(relevant) == 0 { - return 0 - } - n := k - if n > len(ranked) { - n = len(ranked) - } - - dcg := 0.0 - for i := 0; i < n; i++ { - if relevant[ranked[i]] { - dcg += 1.0 / math.Log2(float64(i+2)) - } - } - - idealCount := len(relevant) - if idealCount > k { - idealCount = k - } - idcg := 0.0 - for i := 0; i < idealCount; i++ { - idcg += 1.0 / math.Log2(float64(i+2)) - } - - if idcg == 0 { - return 0 - } - return dcg / idcg -} - -// @intent treat any returned result for a negative query as one false-positive hit in aggregate eval reporting. -func FalsePositiveRate(ranked []string) float64 { - if len(ranked) == 0 { - return 0 - } - return 1 -} diff --git a/internal/eval/metrics_test.go b/internal/eval/metrics_test.go deleted file mode 100644 index 73626d5..0000000 --- a/internal/eval/metrics_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package eval - -import ( - "math" - "testing" -) - -const epsilon = 1e-9 - -func approxEqual(a, b float64) bool { - return math.Abs(a-b) < epsilon -} - -func TestComputeClassification_AllCorrect(t *testing.T) { - expected := []string{"a", "b", "c"} - actual := []string{"a", "b", "c"} - m := ComputeClassification(expected, actual) - - if m.TruePositive != 3 { - t.Errorf("TP: got %d, want 3", m.TruePositive) - } - if m.FalsePositive != 0 { - t.Errorf("FP: got %d, want 0", m.FalsePositive) - } - if m.FalseNegative != 0 { - t.Errorf("FN: got %d, want 0", m.FalseNegative) - } - if !approxEqual(m.Precision, 1.0) { - t.Errorf("Precision: got %f, want 1.0", m.Precision) - } - if !approxEqual(m.Recall, 1.0) { - t.Errorf("Recall: got %f, want 1.0", m.Recall) - } - if !approxEqual(m.F1, 1.0) { - t.Errorf("F1: got %f, want 1.0", m.F1) - } -} - -func TestComputeClassification_PartialMatch(t *testing.T) { - expected := []string{"a", "b", "c"} - actual := []string{"a", "b", "d"} // d is FP, c is FN - m := ComputeClassification(expected, actual) - - if m.TruePositive != 2 { - t.Errorf("TP: got %d, want 2", m.TruePositive) - } - if m.FalsePositive != 1 { - t.Errorf("FP: got %d, want 1", m.FalsePositive) - } - if m.FalseNegative != 1 { - t.Errorf("FN: got %d, want 1", m.FalseNegative) - } - // Precision = 2/3 - if !approxEqual(m.Precision, 2.0/3.0) { - t.Errorf("Precision: got %f, want %f", m.Precision, 2.0/3.0) - } - // Recall = 2/3 - if !approxEqual(m.Recall, 2.0/3.0) { - t.Errorf("Recall: got %f, want %f", m.Recall, 2.0/3.0) - } -} - -func TestComputeClassification_Empty(t *testing.T) { - m := ComputeClassification(nil, nil) - if m.Precision != 0 || m.Recall != 0 || m.F1 != 0 { - t.Errorf("empty should be zeros: %+v", m) - } -} - -func TestComputeClassification_NoOverlap(t *testing.T) { - m := ComputeClassification([]string{"a", "b"}, []string{"c", "d"}) - if m.TruePositive != 0 { - t.Errorf("TP: got %d, want 0", m.TruePositive) - } - if !approxEqual(m.Precision, 0) { - t.Errorf("Precision: got %f, want 0", m.Precision) - } -} - -func TestPrecisionAtK(t *testing.T) { - relevant := map[string]bool{"a": true, "b": true, "c": true} - ranked := []string{"a", "d", "b", "e", "c"} - - if got := PrecisionAtK(ranked, relevant, 1); !approxEqual(got, 1.0) { - t.Errorf("P@1: got %f, want 1.0", got) - } - if got := PrecisionAtK(ranked, relevant, 3); !approxEqual(got, 2.0/3.0) { - t.Errorf("P@3: got %f, want %f", got, 2.0/3.0) - } - if got := PrecisionAtK(ranked, relevant, 5); !approxEqual(got, 3.0/5.0) { - t.Errorf("P@5: got %f, want %f", got, 3.0/5.0) - } -} - -func TestRecallAtK(t *testing.T) { - relevant := map[string]bool{"a": true, "b": true, "c": true} - ranked := []string{"a", "d", "b", "e", "c"} - - if got := RecallAtK(ranked, relevant, 1); !approxEqual(got, 1.0/3.0) { - t.Errorf("R@1: got %f, want %f", got, 1.0/3.0) - } - if got := RecallAtK(ranked, relevant, 5); !approxEqual(got, 1.0) { - t.Errorf("R@5: got %f, want 1.0", got) - } -} - -func TestMRR(t *testing.T) { - relevant := map[string]bool{"b": true} - ranked := []string{"a", "b", "c"} - if got := MRR(ranked, relevant); !approxEqual(got, 0.5) { - t.Errorf("MRR: got %f, want 0.5", got) - } - - // not found - if got := MRR(ranked, map[string]bool{"z": true}); !approxEqual(got, 0) { - t.Errorf("MRR not found: got %f, want 0", got) - } -} - -func TestNDCG(t *testing.T) { - relevant := map[string]bool{"a": true, "b": true, "c": true} - // Perfect ranking - perfect := []string{"a", "b", "c"} - if got := NDCG(perfect, relevant, 3); !approxEqual(got, 1.0) { - t.Errorf("nDCG perfect: got %f, want 1.0", got) - } - - // Worst: no relevant in top K - worst := []string{"x", "y", "z"} - if got := NDCG(worst, relevant, 3); !approxEqual(got, 0) { - t.Errorf("nDCG worst: got %f, want 0", got) - } -} - -func TestPrecisionAtK_EmptyInputs(t *testing.T) { - if got := PrecisionAtK(nil, nil, 5); got != 0 { - t.Errorf("empty: got %f, want 0", got) - } -} - -func TestFalsePositiveRate_EmptyRankedReturnsZero(t *testing.T) { - if got := FalsePositiveRate(nil); got != 0 { - t.Fatalf("nil ranked: got %f, want 0", got) - } - if got := FalsePositiveRate([]string{}); got != 0 { - t.Fatalf("empty ranked: got %f, want 0", got) - } -} - -func TestFalsePositiveRate_NonEmptyRankedReturnsOne(t *testing.T) { - if got := FalsePositiveRate([]string{"x"}); got != 1 { - t.Fatalf("single result: got %f, want 1", got) - } - if got := FalsePositiveRate([]string{"x", "y"}); got != 1 { - t.Fatalf("multiple results: got %f, want 1", got) - } -} diff --git a/internal/eval/parser.go b/internal/eval/parser.go deleted file mode 100644 index d840af4..0000000 --- a/internal/eval/parser.go +++ /dev/null @@ -1,315 +0,0 @@ -// @index Golden corpus loading and normalization helpers for parser evaluation. -package eval - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - - "github.com/tae2089/code-context-graph/internal/model" -) - -// @intent load every language-specific golden corpus file from the eval corpus directory tree. -func LoadGoldenDir(dir string) ([]GoldenCorpus, error) { - var corpora []GoldenCorpus - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - langDir := filepath.Join(dir, entry.Name()) - files, err := os.ReadDir(langDir) - if err != nil { - return nil, err - } - for _, f := range files { - if f.IsDir() || !strings.HasSuffix(f.Name(), ".golden.json") { - continue - } - data, err := os.ReadFile(filepath.Join(langDir, f.Name())) - if err != nil { - return nil, err - } - var c GoldenCorpus - if err := json.Unmarshal(data, &c); err != nil { - return nil, err - } - if c.Language == "" { - c.Language = entry.Name() - } - corpora = append(corpora, c) - } - } - return corpora, nil -} - -// @intent persist normalized parser output as a golden snapshot for future comparisons. -func WriteGolden(path string, corpus GoldenCorpus) error { - data, err := json.MarshalIndent(corpus, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, append(data, '\n'), 0o644) -} - -// @intent project eval nodes into stable comparison keys for set-based metrics. -func NodeKeys(nodes []EvalNode) []string { - keys := make([]string, len(nodes)) - for i, n := range nodes { - keys[i] = n.Key() - } - return keys -} - -// @intent project eval edges into stable comparison keys for set-based metrics. -func EdgeKeys(edges []EvalEdge) []string { - keys := make([]string, len(edges)) - for i, e := range edges { - keys[i] = e.Key() - } - return keys -} - -// @intent summarize parser accuracy for one language corpus by comparing expected and actual nodes and edges. -func CompareCorpus(expected, actual GoldenCorpus) LanguageReport { - nodeMetrics := ComputeClassification(NodeKeys(expected.Nodes), NodeKeys(actual.Nodes)) - edgeMetrics := ComputeClassification(EdgeKeys(expected.Edges), EdgeKeys(actual.Edges)) - return LanguageReport{ - Language: expected.Language, - NodeMetrics: nodeMetrics, - EdgeMetrics: edgeMetrics, - Files: 1, - } -} - -// @intent normalize parsed graph nodes into corpus-stable eval records independent of absolute paths. -func NormalizeNodes(nodes []model.Node, baseDir string) []EvalNode { - out := make([]EvalNode, 0, len(nodes)) - for _, n := range nodes { - if n.Kind == model.NodeKindFile { - continue - } - file := n.FilePath - if baseDir != "" { - file, _ = filepath.Rel(baseDir, file) - } - out = append(out, EvalNode{ - ID: n.QualifiedName, - Kind: string(n.Kind), - Name: n.Name, - File: file, - StartLine: n.StartLine, - EndLine: n.EndLine, - }) - } - return out -} - -// @intent normalize parsed graph edges into corpus-stable eval records keyed by qualified names. -func NormalizeEdges(edges []model.Edge, nodes []model.Node) []EvalEdge { - out := make([]EvalEdge, 0, len(edges)) - nodesByQName := make(map[string]model.Node, len(nodes)) - for _, n := range nodes { - nodesByQName[n.QualifiedName] = n - } - for _, e := range edges { - from, to := normalizeEdgeEndpoints(e, nodesByQName) - if from == "" || to == "" { - continue - } - out = append(out, EvalEdge{ - Kind: string(e.Kind), - From: from, - To: to, - }) - } - return out -} - -// normalizeEdgeEndpoints reconstructs stable edge endpoints from parser fingerprints. -// @intent eval용 edge 비교에서 파일 경로와 fingerprint를 corpus-stable 식별자로 바꾼다. -func normalizeEdgeEndpoints(edge model.Edge, nodesByQName map[string]model.Node) (string, string) { - parts := strings.Split(edge.Fingerprint, ":") - if len(parts) < 2 { - return "", "" - } - - kind := parts[0] - switch kind { - case string(model.EdgeKindContains): - if target, ok := containsTarget(edge); ok { - return edge.FilePath, target - } - case string(model.EdgeKindCalls), string(model.EdgeKindFallbackCalls), string(model.EdgeKindImportsFrom): - if len(parts) >= 4 { - if kind == string(model.EdgeKindImportsFrom) { - if target, ok := importsFromTarget(edge); ok { - return edge.FilePath, target - } - return edge.FilePath, "" - } - // fallback_calls should be compared and normalized identically to calls. - if target, ok := callsTarget(edge); ok { - from := resolveParserStageOwner(edge, nodesByQName) - return from, target - } - } - case string(model.EdgeKindTestedBy): - if len(parts) >= 4 { - from := parts[len(parts)-1] - to := strings.Join(parts[2:len(parts)-1], ":") - if from == "" || to == "" { - return "", "" - } - return from, to - } - case string(model.EdgeKindImplements): - if len(parts) >= 4 { - prefix := "implements:" + edge.FilePath + ":" - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", "" - } - rest := strings.TrimPrefix(edge.Fingerprint, prefix) - from, to, ok := strings.Cut(rest, ":") - if !ok { - return "", "" - } - if from == "" || to == "" { - return "", "" - } - return from, to - } - case string(model.EdgeKindInherits): - from, to, ok := model.ParseInheritsFingerprint(edge.FilePath, edge.Fingerprint) - if ok && from != "" && to != "" { - return from, to - } - } - return "", "" -} - -// importsFromTarget extracts the import target from an imports_from fingerprint. -// @intent import edge fingerprint를 eval 비교에 필요한 target 문자열로 복원한다. -func importsFromTarget(edge model.Edge) (string, bool) { - prefix := "imports_from:" + edge.FilePath + ":" - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", false - } - rest := strings.TrimPrefix(edge.Fingerprint, prefix) - idx := strings.LastIndex(rest, ":") - if idx < 0 { - return "", false - } - target := rest[:idx] - return target, target != "" -} - -// callsTarget extracts the callee target from a calls or fallback_calls fingerprint. -// @intent call-like edge fingerprint에서 target과 trailing line 구문이 유효한지 함께 확인한다. -func callsTarget(edge model.Edge) (string, bool) { - var prefix string - switch edge.Kind { - case model.EdgeKindCalls: - prefix = "calls:" + edge.FilePath + ":" - case model.EdgeKindFallbackCalls: - prefix = "fallback_calls:" + edge.FilePath + ":" - default: - return "", false - } - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", false - } - rest := strings.TrimPrefix(edge.Fingerprint, prefix) - idx := strings.LastIndex(rest, ":") - if idx < 0 { - return "", false - } - target := rest[:idx] - if target == "" { - return "", false - } - if _, err := parseTrailingLine(rest[idx+1:]); err != nil { - return "", false - } - return target, true -} - -// containsTarget extracts the contained symbol target from a contains fingerprint. -// @intent contains edge fingerprint를 eval 비교용 대상 심볼 이름으로 복원한다. -func containsTarget(edge model.Edge) (string, bool) { - prefix := "contains:" + edge.FilePath + ":" - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", false - } - return strings.TrimPrefix(edge.Fingerprint, prefix), true -} - -// parseTrailingLine parses the trailing source line segment embedded in a fingerprint. -// @intent malformed fingerprint를 조기에 배제해 eval edge 비교 안정성을 높인다. -func parseTrailingLine(s string) (int, error) { - if s == "" { - return 0, os.ErrInvalid - } - line := 0 - for _, r := range s { - if r < '0' || r > '9' { - return 0, os.ErrInvalid - } - line = line*10 + int(r-'0') - } - if line <= 0 { - return 0, os.ErrInvalid - } - return line, nil -} - -// resolveParserStageOwner picks the best owner node for a parser-stage edge. -// @intent line 정보 또는 파일 내 첫 심볼을 이용해 parser fingerprint의 from endpoint를 정한다. -func resolveParserStageOwner(edge model.Edge, nodesByQName map[string]model.Node) string { - if edge.Line > 0 { - if owner := resolveOwnerByLine(edge.FilePath, edge.Line, nodesByQName); owner != "" { - return owner - } - } - for _, n := range nodesByQName { - if n.FilePath == edge.FilePath && n.Kind != model.NodeKindFile { - return n.QualifiedName - } - } - return "" -} - -// resolveOwnerByLine finds the narrowest symbol spanning a given file line. -// @intent parser-stage edge가 속한 가장 구체적인 owner 심볼을 line 범위로 찾는다. -func resolveOwnerByLine(filePath string, line int, nodesByQName map[string]model.Node) string { - var best model.Node - bestFound := false - for _, n := range nodesByQName { - if n.FilePath != filePath || n.Kind == model.NodeKindFile { - continue - } - if n.StartLine <= line && line <= n.EndLine { - if !bestFound || span(n) < span(best) || (span(n) == span(best) && n.StartLine > best.StartLine) { - best = n - bestFound = true - } - } - } - if bestFound { - return best.QualifiedName - } - return "" -} - -// span returns the inclusive line span width for a node. -// @intent owner 선택 시 더 좁은 범위의 심볼을 우선하도록 비교 지표를 제공한다. -func span(n model.Node) int { - if n.EndLine < n.StartLine { - return 0 - } - return n.EndLine - n.StartLine -} diff --git a/internal/eval/parser_test.go b/internal/eval/parser_test.go deleted file mode 100644 index ca198ff..0000000 --- a/internal/eval/parser_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package eval - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/tae2089/code-context-graph/internal/model" -) - -func TestLoadGoldenCorpus(t *testing.T) { - dir := t.TempDir() - - corpus := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - }, - Edges: []EvalEdge{ - {Kind: "calls", From: "n1", To: "n2"}, - }, - } - - data, _ := json.MarshalIndent(corpus, "", " ") - langDir := filepath.Join(dir, "go") - os.MkdirAll(langDir, 0o755) - os.WriteFile(filepath.Join(langDir, "sample.golden.json"), data, 0o644) - - loaded, err := LoadGoldenDir(dir) - if err != nil { - t.Fatalf("LoadGoldenDir: %v", err) - } - if len(loaded) != 1 { - t.Fatalf("got %d corpora, want 1", len(loaded)) - } - if loaded[0].Language != "go" { - t.Errorf("language: got %s, want go", loaded[0].Language) - } - if len(loaded[0].Nodes) != 1 { - t.Errorf("nodes: got %d, want 1", len(loaded[0].Nodes)) - } -} - -func TestNormalizeNodes(t *testing.T) { - nodes := []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - {ID: "n2", Kind: "class", Name: "World", File: "sample.go", StartLine: 10}, - } - keys := NodeKeys(nodes) - if len(keys) != 2 { - t.Fatalf("got %d keys, want 2", len(keys)) - } - if keys[0] != "function:Hello@sample.go" { - t.Errorf("key[0]: got %s", keys[0]) - } -} - -func TestCompareCorpus(t *testing.T) { - expected := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - {ID: "n2", Kind: "function", Name: "World", File: "sample.go", StartLine: 10}, - }, - Edges: []EvalEdge{ - {Kind: "calls", From: "n1", To: "n2"}, - }, - } - - actual := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - {ID: "n3", Kind: "function", Name: "Extra", File: "sample.go", StartLine: 20}, - }, - Edges: []EvalEdge{ - {Kind: "calls", From: "n1", To: "n2"}, - }, - } - - report := CompareCorpus(expected, actual) - if report.NodeMetrics.TruePositive != 1 { - t.Errorf("node TP: got %d, want 1", report.NodeMetrics.TruePositive) - } - if report.NodeMetrics.FalsePositive != 1 { - t.Errorf("node FP: got %d, want 1", report.NodeMetrics.FalsePositive) - } - if report.NodeMetrics.FalseNegative != 1 { - t.Errorf("node FN: got %d, want 1", report.NodeMetrics.FalseNegative) - } - if report.EdgeMetrics.TruePositive != 1 { - t.Errorf("edge TP: got %d, want 1", report.EdgeMetrics.TruePositive) - } -} - -func TestUpdateGolden(t *testing.T) { - dir := t.TempDir() - langDir := filepath.Join(dir, "go") - os.MkdirAll(langDir, 0o755) - - corpus := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - }, - } - - outPath := filepath.Join(langDir, "sample.golden.json") - err := WriteGolden(outPath, corpus) - if err != nil { - t.Fatalf("WriteGolden: %v", err) - } - - loaded, err := LoadGoldenDir(dir) - if err != nil { - t.Fatalf("reload: %v", err) - } - if len(loaded) != 1 || loaded[0].Nodes[0].Name != "Hello" { - t.Errorf("round-trip failed: %+v", loaded) - } -} - -func TestNormalizeEdges_PreservesParserStageEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "sample.Hello", Kind: model.NodeKindFunction, Name: "Hello", FilePath: "sample.go", StartLine: 3, EndLine: 5}, - {ID: 0, QualifiedName: "sample.World", Kind: model.NodeKindFunction, Name: "World", FilePath: "sample.go", StartLine: 10, EndLine: 12}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindCalls, FilePath: "sample.go", Line: 4, Fingerprint: "calls:sample.go:sample.World:4"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "sample.Hello" { - t.Fatalf("from collapsed: got %q, want %q", actual[0].From, "sample.Hello") - } - if actual[0].To != "sample.World" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "sample.World") - } -} - -func TestNormalizeEdges_PreservesFallbackCallEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "sample.Hello", Kind: model.NodeKindFunction, Name: "Hello", FilePath: "sample.go", StartLine: 3, EndLine: 5}, - {ID: 0, QualifiedName: "sample.World", Kind: model.NodeKindFunction, Name: "World", FilePath: "sample.go", StartLine: 10, EndLine: 12}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindFallbackCalls, FilePath: "sample.go", Line: 4, Fingerprint: "fallback_calls:sample.go:sample.World:4"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "sample.Hello" { - t.Fatalf("from collapsed: got %q, want %q", actual[0].From, "sample.Hello") - } - if actual[0].To != "sample.World" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "sample.World") - } -} - -func TestNormalizeEdges_ImportsFromUsesFilePathAndFullTarget(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.File", Kind: model.NodeKindFile, Name: "File", FilePath: "dir:with:colon/sample.go"}, - {ID: 0, QualifiedName: "pkg.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "dir:with:colon/sample.go", StartLine: 3, EndLine: 5}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindImportsFrom, FilePath: "dir:with:colon/sample.go", Line: 4, Fingerprint: "imports_from:dir:with:colon/sample.go:github.com/acme/lib:v2/util:4"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "dir:with:colon/sample.go" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "dir:with:colon/sample.go") - } - if actual[0].To != "github.com/acme/lib:v2/util" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util") - } -} - -func TestNormalizeEdges_TestedByUsesTestQNameAsFrom(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "sample.TestHello", Kind: model.NodeKindFunction, Name: "TestHello", FilePath: "sample_test.go", StartLine: 3, EndLine: 5}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "sample.go", StartLine: 10, EndLine: 12}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindTestedBy, FilePath: "sample_test.go", Line: 4, Fingerprint: "tested_by:sample_test.go:github.com/acme/lib:v2/util.Helper:sample.TestHello"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "sample.TestHello" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "sample.TestHello") - } - if actual[0].To != "github.com/acme/lib:v2/util.Helper" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Helper") - } -} - -func TestNormalizeEdges_ImplementsUsesFingerprintEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Interface", Kind: model.NodeKindClass, Name: "Interface", FilePath: "iface.go", StartLine: 10, EndLine: 12}, - {ID: 0, QualifiedName: "pkg.Implementation", Kind: model.NodeKindClass, Name: "Implementation", FilePath: "impl.go", StartLine: 20, EndLine: 24}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindImplements, FilePath: "parser.go", Line: 0, Fingerprint: "implements:parser.go:pkg.Implementation:pkg.Interface"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "pkg.Implementation" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "pkg.Implementation") - } - if actual[0].To != "pkg.Interface" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "pkg.Interface") - } -} - -func TestNormalizeEdges_InheritsUsesFingerprintEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Child", Kind: model.NodeKindClass, Name: "Child", FilePath: "models.py", StartLine: 10, EndLine: 20}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Base", Kind: model.NodeKindClass, Name: "Base", FilePath: "models.py", StartLine: 30, EndLine: 40}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindInherits, FilePath: "models.py", Line: 0, Fingerprint: model.BuildInheritsFingerprintV2("models.py", "pkg.Child", "github.com/acme/lib:v2/util.Base")}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "pkg.Child" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "pkg.Child") - } - if actual[0].To != "github.com/acme/lib:v2/util.Base" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Base") - } -} - -func TestNormalizeEdges_CallsUsesLastColonSafeTargetAndNumericLine(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Owner", Kind: model.NodeKindFunction, Name: "Owner", FilePath: "dir:with:colon/parser.go", StartLine: 10, EndLine: 20}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "dir:with:colon/parser.go", StartLine: 30, EndLine: 40}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindCalls, FilePath: "dir:with:colon/parser.go", Line: 12, Fingerprint: "calls:dir:with:colon/parser.go:github.com/acme/lib:v2/util.Helper:12"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "pkg.Owner" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "pkg.Owner") - } - if actual[0].To != "github.com/acme/lib:v2/util.Helper" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Helper") - } -} - -func TestNormalizeEdges_ContainsUsesFullTargetAfterFilePathPrefix(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Owner", Kind: model.NodeKindFunction, Name: "Owner", FilePath: "dir:with:colon/parser.go", StartLine: 10, EndLine: 20}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "dir:with:colon/parser.go", StartLine: 30, EndLine: 40}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindContains, FilePath: "dir:with:colon/parser.go", Line: 11, Fingerprint: "contains:dir:with:colon/parser.go:github.com/acme/lib:v2/util.Helper"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "dir:with:colon/parser.go" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "dir:with:colon/parser.go") - } - if actual[0].To != "github.com/acme/lib:v2/util.Helper" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Helper") - } -} diff --git a/internal/eval/runner.go b/internal/eval/runner.go deleted file mode 100644 index 8984c1a..0000000 --- a/internal/eval/runner.go +++ /dev/null @@ -1,232 +0,0 @@ -// @index Eval runner orchestration for parser and search benchmark suites. -package eval - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/tae2089/code-context-graph/internal/parse/treesitter" -) - -// RunOptions collects corpus paths, parser walkers, and output settings for one eval invocation. -// @intent configure parser/search evaluation suites from CLI without leaking command details. -type RunOptions struct { - CorpusDir string - Suite string - Format string - Update bool - Walkers map[string]*treesitter.Walker - SearchFn SearchFunc - Writer io.Writer -} - -// Run executes the requested evaluation suites and writes the chosen report format. -// @intent provide one orchestration entry point for CLI-driven parser and search evaluation. -func Run(ctx context.Context, opts RunOptions) (*Report, error) { - report := &Report{Suite: opts.Suite} - - if opts.Suite == "all" || opts.Suite == "parser" { - if err := runParserEval(ctx, opts, report); err != nil { - return nil, fmt.Errorf("parser eval: %w", err) - } - } - - if opts.Suite == "all" || opts.Suite == "search" { - if err := runSearchEval(ctx, opts, report); err != nil { - return nil, fmt.Errorf("search eval: %w", err) - } - } - - if opts.Format == "json" { - return report, writeJSON(opts.Writer, report) - } - return report, writeTable(opts.Writer, report) -} - -// @intent route parser evaluation into update or comparison mode without duplicating top-level flow control. -func runParserEval(ctx context.Context, opts RunOptions, report *Report) error { - if opts.Update { - return runParserUpdate(ctx, opts) - } - return runParserCompare(ctx, opts, report) -} - -// @intent regenerate golden parser snapshots from the current parser implementation. -func runParserUpdate(ctx context.Context, opts RunOptions) error { - entries, err := os.ReadDir(opts.CorpusDir) - if err != nil { - return err - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - lang := entry.Name() - walker, ok := opts.Walkers[lang] - if !ok { - continue - } - langDir := filepath.Join(opts.CorpusDir, lang) - files, err := os.ReadDir(langDir) - if err != nil { - return err - } - for _, f := range files { - if f.IsDir() || strings.HasSuffix(f.Name(), ".golden.json") { - continue - } - srcFile := filepath.Join(langDir, f.Name()) - src, err := readFileContent(srcFile) - if err != nil { - return fmt.Errorf("read source %s: %w", srcFile, err) - } - nodes, edges, _, err := walker.ParseWithComments(ctx, srcFile, src) - if err != nil { - return fmt.Errorf("parse %s: %w", srcFile, err) - } - actualNodes := NormalizeNodes(nodes, langDir) - actualEdges := NormalizeEdges(edges, nodes) - - golden := GoldenCorpus{ - Language: lang, - File: f.Name(), - Nodes: actualNodes, - Edges: actualEdges, - } - goldenPath := filepath.Join(langDir, f.Name()+".golden.json") - if err := WriteGolden(goldenPath, golden); err != nil { - return fmt.Errorf("write golden %s: %w", goldenPath, err) - } - } - } - return nil -} - -// @intent compare current parser output against stored golden corpora and accumulate language reports. -func runParserCompare(ctx context.Context, opts RunOptions, report *Report) error { - corpora, err := LoadGoldenDir(opts.CorpusDir) - if err != nil { - return err - } - - byLang := make(map[string][]GoldenCorpus) - for _, c := range corpora { - byLang[c.Language] = append(byLang[c.Language], c) - } - - for lang, goldens := range byLang { - walker, ok := opts.Walkers[lang] - if !ok { - continue - } - - var allExpectedNodes, allActualNodes []string - var allExpectedEdges, allActualEdges []string - fileCount := 0 - - for _, golden := range goldens { - langDir := filepath.Join(opts.CorpusDir, lang) - srcFile := filepath.Join(langDir, golden.File) - src, err := readFileContent(srcFile) - if err != nil { - return fmt.Errorf("read source %s: %w", srcFile, err) - } - - nodes, edges, _, err := walker.ParseWithComments(ctx, srcFile, src) - if err != nil { - return fmt.Errorf("parse %s: %w", srcFile, err) - } - - actualNodes := NormalizeNodes(nodes, langDir) - actualEdges := NormalizeEdges(edges, nodes) - - allExpectedNodes = append(allExpectedNodes, NodeKeys(golden.Nodes)...) - allActualNodes = append(allActualNodes, NodeKeys(actualNodes)...) - allExpectedEdges = append(allExpectedEdges, EdgeKeys(golden.Edges)...) - allActualEdges = append(allActualEdges, EdgeKeys(actualEdges)...) - fileCount++ - } - - report.Languages = append(report.Languages, LanguageReport{ - Language: lang, - NodeMetrics: ComputeClassification(allExpectedNodes, allActualNodes), - EdgeMetrics: ComputeClassification(allExpectedEdges, allActualEdges), - Files: fileCount, - }) - } - - return nil -} - -// @intent run ranking metrics over the search corpus when a search backend is available. -func runSearchEval(_ context.Context, opts RunOptions, report *Report) error { - queryPath := fmt.Sprintf("%s/queries.json", opts.CorpusDir) - qc, err := LoadQueryCorpus(queryPath) - if err != nil { - return nil - } - - if opts.SearchFn == nil { - return fmt.Errorf("search function not configured") - } - - sr, err := EvaluateQueries(qc.Queries, opts.SearchFn) - if err != nil { - return err - } - report.Search = &sr - return nil -} - -// @intent centralize file reads used by parser evaluation so tests can characterize failures consistently. -func readFileContent(path string) ([]byte, error) { - return os.ReadFile(path) -} - -// @intent emit the eval report as pretty-printed JSON for machine consumption. -func writeJSON(w io.Writer, report *Report) error { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(report) -} - -// @intent render a compact human-readable summary of parser and search evaluation metrics. -func writeTable(w io.Writer, report *Report) error { - if len(report.Languages) > 0 { - fmt.Fprintf(w, "=== Parser Evaluation ===\n\n") - fmt.Fprintf(w, "%-14s %6s %8s %8s %8s %8s %8s %8s\n", - "Language", "Files", "Node P", "Node R", "Node F1", "Edge P", "Edge R", "Edge F1") - fmt.Fprintf(w, "%s\n", strings.Repeat("─", 82)) - - for _, lr := range report.Languages { - fmt.Fprintf(w, "%-14s %6d %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f\n", - lr.Language, lr.Files, - lr.NodeMetrics.Precision, lr.NodeMetrics.Recall, lr.NodeMetrics.F1, - lr.EdgeMetrics.Precision, lr.EdgeMetrics.Recall, lr.EdgeMetrics.F1) - } - fmt.Fprintln(w) - } - - if report.Search != nil { - fmt.Fprintf(w, "=== Search Evaluation ===\n\n") - fmt.Fprintf(w, "Queries: %d\n", report.Search.QueriesTotal) - fmt.Fprintf(w, "P@1: %.4f\n", report.Search.AvgPAt1) - fmt.Fprintf(w, "P@3: %.4f\n", report.Search.AvgPAt3) - fmt.Fprintf(w, "P@5: %.4f\n", report.Search.AvgPAt5) - fmt.Fprintf(w, "R@5: %.4f\n", report.Search.AvgRecallAt5) - fmt.Fprintf(w, "MRR: %.4f\n", report.Search.AvgMRR) - fmt.Fprintf(w, "nDCG@5: %.4f\n", report.Search.AvgNDCGAt5) - if report.Search.NegativeQueries > 0 { - fmt.Fprintf(w, "Negatives: %d\n", report.Search.NegativeQueries) - fmt.Fprintf(w, "FP: %d\n", report.Search.NegativeFalsePositives) - fmt.Fprintf(w, "Pass Rate: %.4f\n", report.Search.NegativePassRate) - } - } - - return nil -} diff --git a/internal/eval/runner_test.go b/internal/eval/runner_test.go deleted file mode 100644 index 821880e..0000000 --- a/internal/eval/runner_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package eval - -import ( - "bytes" - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/parse/treesitter" -) - -func TestRunParserEval_Update(t *testing.T) { - walker := treesitter.NewWalker(treesitter.GoSpec) - - dir := t.TempDir() - copyTestCorpus(t, dir, "go", "sample.go", `package sample - -func Hello() {} - -func World() { - Hello() -} -`) - - var buf bytes.Buffer - opts := RunOptions{ - CorpusDir: dir, - Suite: "parser", - Format: "table", - Update: true, - Walkers: map[string]*treesitter.Walker{"go": walker}, - Writer: &buf, - } - - _, err := Run(context.Background(), opts) - if err != nil { - t.Fatalf("Run update: %v", err) - } - - corpora, err := LoadGoldenDir(dir) - if err != nil { - t.Fatalf("LoadGoldenDir: %v", err) - } - if len(corpora) != 1 { - t.Fatalf("corpora: got %d, want 1", len(corpora)) - } - - found := false - for _, n := range corpora[0].Nodes { - if n.Name == "Hello" && n.Kind == "function" { - found = true - } - } - if !found { - t.Errorf("expected Hello function in golden, got: %+v", corpora[0].Nodes) - } -} - -func TestRunParserEval_Compare(t *testing.T) { - walker := treesitter.NewWalker(treesitter.GoSpec) - - dir := t.TempDir() - copyTestCorpus(t, dir, "go", "sample.go", `package sample - -func Hello() {} - -func World() { - Hello() -} -`) - - walkers := map[string]*treesitter.Walker{"go": walker} - - var buf bytes.Buffer - _, err := Run(context.Background(), RunOptions{ - CorpusDir: dir, Suite: "parser", Format: "table", - Update: true, Walkers: walkers, Writer: &buf, - }) - if err != nil { - t.Fatalf("update: %v", err) - } - - buf.Reset() - report, err := Run(context.Background(), RunOptions{ - CorpusDir: dir, Suite: "parser", Format: "table", - Walkers: walkers, Writer: &buf, - }) - if err != nil { - t.Fatalf("compare: %v", err) - } - - if len(report.Languages) != 1 { - t.Fatalf("languages: got %d, want 1", len(report.Languages)) - } - - lr := report.Languages[0] - if !approxEqual(lr.NodeMetrics.F1, 1.0) { - t.Errorf("node F1: got %f, want 1.0", lr.NodeMetrics.F1) - } - if buf.Len() == 0 { - t.Error("expected table output") - } -} - -func copyTestCorpus(t *testing.T, dir, lang, filename, content string) { - t.Helper() - langDir := filepath.Join(dir, lang) - os.MkdirAll(langDir, 0o755) - os.WriteFile(filepath.Join(langDir, filename), []byte(content), 0o644) -} - -func TestWriteTable_RendersNegativeControlBlock(t *testing.T) { - var buf bytes.Buffer - report := &Report{Search: &SearchReport{ - QueriesTotal: 2, - AvgPAt1: 1.0, - AvgPAt3: 0.8333, - AvgPAt5: 0.8, - AvgRecallAt5: 1.0, - AvgMRR: 1.0, - AvgNDCGAt5: 1.0, - NegativeQueries: 1, - NegativeFalsePositives: 0, - NegativePassRate: 1.0, - }} - - if err := writeTable(&buf, report); err != nil { - t.Fatalf("writeTable: %v", err) - } - out := buf.String() - for _, want := range []string{"Negatives:", "FP:", "Pass Rate: 1.0000"} { - if !strings.Contains(out, want) { - t.Fatalf("expected output to contain %q, got:\n%s", want, out) - } - } -} - -func TestWriteTable_OmitsNegativeBlock_WhenZeroNegatives(t *testing.T) { - var buf bytes.Buffer - report := &Report{Search: &SearchReport{ - QueriesTotal: 1, - AvgPAt1: 1.0, - AvgPAt3: 1.0, - AvgPAt5: 1.0, - AvgRecallAt5: 1.0, - AvgMRR: 1.0, - AvgNDCGAt5: 1.0, - NegativeQueries: 0, - }} - - if err := writeTable(&buf, report); err != nil { - t.Fatalf("writeTable: %v", err) - } - if strings.Contains(buf.String(), "Negatives:") { - t.Fatalf("did not expect negative block, got:\n%s", buf.String()) - } -} diff --git a/internal/eval/schema.go b/internal/eval/schema.go deleted file mode 100644 index 551f04f..0000000 --- a/internal/eval/schema.go +++ /dev/null @@ -1,105 +0,0 @@ -// @index Data schemas for parser golden corpora and search evaluation reports. -package eval - -// EvalNode is the normalized parser node shape used in golden corpora. -// @intent capture corpus-stable node identity independent of absolute paths. -type EvalNode struct { - ID string `json:"id"` - Kind string `json:"kind"` - Name string `json:"name"` - File string `json:"file"` - StartLine int `json:"start_line"` - EndLine int `json:"end_line,omitempty"` -} - -// Key derives a stable string key for node-level evaluation comparisons. -// @intent provide a deterministic identifier for set-based parser metrics. -func (n EvalNode) Key() string { - return n.Kind + ":" + n.Name + "@" + n.File -} - -// EvalEdge is the normalized parser edge shape used in golden corpora. -// @intent capture corpus-stable edge identity for parser comparisons. -type EvalEdge struct { - Kind string `json:"kind"` - From string `json:"from"` - To string `json:"to"` -} - -// Key derives a stable string key for edge-level evaluation comparisons. -// @intent provide a deterministic identifier for set-based parser metrics. -func (e EvalEdge) Key() string { - return e.Kind + ":" + e.From + "->" + e.To -} - -// GoldenCorpus stores one parser snapshot for a source file in a language corpus. -// @intent persist expected parser output for regression comparison. -type GoldenCorpus struct { - Language string `json:"language"` - File string `json:"file"` - Nodes []EvalNode `json:"nodes"` - Edges []EvalEdge `json:"edges"` -} - -// QueryCase defines one search evaluation query and its relevant expected results. -// @intent describe a single ranked-retrieval test case for search evaluation. -type QueryCase struct { - Query string `json:"query"` - Relevant []string `json:"relevant"` - K int `json:"k,omitempty"` -} - -// QueryCorpus groups search evaluation cases loaded from one corpus directory. -// @intent represent the full search evaluation suite as one loadable artifact. -type QueryCorpus struct { - CorpusDir string `json:"corpus_dir"` - Queries []QueryCase `json:"queries"` -} - -// LanguageReport summarizes parser accuracy metrics for one language corpus. -// @intent expose per-language node and edge metrics in the eval report. -type LanguageReport struct { - Language string `json:"language"` - NodeMetrics ClassificationMetrics `json:"node_metrics"` - EdgeMetrics ClassificationMetrics `json:"edge_metrics"` - Files int `json:"files"` -} - -// SearchReport holds aggregate ranking metrics across the search evaluation corpus. -// @intent surface average P@K, recall, MRR, and nDCG over all search queries. -type SearchReport struct { - QueriesTotal int `json:"queries_total"` - AvgPAt1 float64 `json:"avg_p_at_1"` - AvgPAt3 float64 `json:"avg_p_at_3"` - AvgPAt5 float64 `json:"avg_p_at_5"` - AvgRecallAt5 float64 `json:"avg_recall_at_5"` - AvgMRR float64 `json:"avg_mrr"` - AvgNDCGAt5 float64 `json:"avg_ndcg_at_5"` - NegativeQueries int `json:"negative_queries"` - NegativeFalsePositives int `json:"negative_false_positives"` - NegativePassRate float64 `json:"negative_pass_rate"` - PerQuery []QueryDiagnostic `json:"per_query,omitempty"` -} - -// QueryDiagnostic records one query's detailed evaluation outcome for debugging regressions. -// @intent aggregate metric만으로 보이지 않는 개별 쿼리 실패 원인을 보존한다. -type QueryDiagnostic struct { - Query string `json:"query"` - Kind string `json:"kind"` - ResultsReturned int `json:"results_returned"` - PAt1 float64 `json:"p_at_1,omitempty"` - PAt5 float64 `json:"p_at_5,omitempty"` - RecallAt5 float64 `json:"recall_at_5,omitempty"` - MRR float64 `json:"mrr,omitempty"` - NDCGAt5 float64 `json:"ndcg_at_5,omitempty"` - FalsePositive bool `json:"false_positive,omitempty"` - TopResults []string `json:"top_results,omitempty"` -} - -// Report bundles parser and search evaluation output into one CLI-facing payload. -// @intent represent the complete eval result returned to CLI and JSON consumers. -type Report struct { - Suite string `json:"suite"` - Languages []LanguageReport `json:"languages,omitempty"` - Search *SearchReport `json:"search,omitempty"` -} diff --git a/internal/eval/search.go b/internal/eval/search.go deleted file mode 100644 index 9241499..0000000 --- a/internal/eval/search.go +++ /dev/null @@ -1,124 +0,0 @@ -// @index Search evaluation corpus loading and ranking metric orchestration. -package eval - -import ( - "encoding/json" - "os" -) - -// capResults truncates a result slice to at most n items. -// @intent per-query 진단에서 상위 결과만 제한해 출력 부피를 제어한다. -func capResults(results []string, n int) []string { - if len(results) <= n { - return results - } - return results[:n] -} - -// SearchFunc abstracts search execution so evaluation can run against any backend. -// @intent decouple ranking metrics from concrete search implementations. -type SearchFunc func(query string, limit int) ([]string, error) - -// LoadQueryCorpus loads search evaluation cases from a JSON corpus file. -// @intent ingest the search query corpus before running ranking evaluation. -func LoadQueryCorpus(path string) (QueryCorpus, error) { - data, err := os.ReadFile(path) - if err != nil { - return QueryCorpus{}, err - } - var qc QueryCorpus - if err := json.Unmarshal(data, &qc); err != nil { - return QueryCorpus{}, err - } - return qc, nil -} - -// @intent execute ranked retrieval metrics across all search evaluation cases. -func EvaluateQueries(cases []QueryCase, searchFn SearchFunc) (SearchReport, error) { - if len(cases) == 0 { - return SearchReport{}, nil - } - - report := SearchReport{QueriesTotal: len(cases)} - var sumP1, sumP3, sumP5, sumR5, sumMRR, sumNDCG5 float64 - negativeQueries := 0 - negativeFalsePositives := 0 - positiveQueries := 0 - - for _, qc := range cases { - k := qc.K - if k <= 0 { - k = 5 - } - limit := k - if limit < 5 { - limit = 5 - } - - results, err := searchFn(qc.Query, limit) - if err != nil { - return SearchReport{}, err - } - - if len(qc.Relevant) == 0 { - negativeQueries++ - falsePositive := FalsePositiveRate(results) > 0 - negativeFalsePositives += int(FalsePositiveRate(results)) - report.PerQuery = append(report.PerQuery, QueryDiagnostic{ - Query: qc.Query, - Kind: "negative", - ResultsReturned: len(results), - FalsePositive: falsePositive, - TopResults: capResults(results, 5), - }) - continue - } - positiveQueries++ - - relevant := make(map[string]bool, len(qc.Relevant)) - for _, r := range qc.Relevant { - relevant[r] = true - } - - p1 := PrecisionAtK(results, relevant, 1) - p3 := PrecisionAtK(results, relevant, 3) - p5 := PrecisionAtK(results, relevant, 5) - r5 := RecallAtK(results, relevant, 5) - mrr := MRR(results, relevant) - ndcg5 := NDCG(results, relevant, 5) - - sumP1 += p1 - sumP3 += p3 - sumP5 += p5 - sumR5 += r5 - sumMRR += mrr - sumNDCG5 += ndcg5 - report.PerQuery = append(report.PerQuery, QueryDiagnostic{ - Query: qc.Query, - Kind: "positive", - ResultsReturned: len(results), - PAt1: p1, - PAt5: p5, - RecallAt5: r5, - MRR: mrr, - NDCGAt5: ndcg5, - TopResults: capResults(results, 5), - }) - } - - report.NegativeQueries = negativeQueries - report.NegativeFalsePositives = negativeFalsePositives - if positiveQueries > 0 { - n := float64(positiveQueries) - report.AvgPAt1 = sumP1 / n - report.AvgPAt3 = sumP3 / n - report.AvgPAt5 = sumP5 / n - report.AvgRecallAt5 = sumR5 / n - report.AvgMRR = sumMRR / n - report.AvgNDCGAt5 = sumNDCG5 / n - } - if negativeQueries > 0 { - report.NegativePassRate = 1 - float64(negativeFalsePositives)/float64(negativeQueries) - } - return report, nil -} diff --git a/internal/eval/search_test.go b/internal/eval/search_test.go deleted file mode 100644 index f62a0f6..0000000 --- a/internal/eval/search_test.go +++ /dev/null @@ -1,451 +0,0 @@ -package eval - -import ( - "encoding/json" - "os" - "path/filepath" - "runtime" - "slices" - "testing" -) - -func repoRoot(t *testing.T) string { - t.Helper() - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) -} - -func TestLoadQueryCorpus(t *testing.T) { - dir := t.TempDir() - qc := QueryCorpus{ - CorpusDir: "testdata/eval/go", - Queries: []QueryCase{ - {Query: "Hello", Relevant: []string{"function:Hello@sample.go"}, K: 5}, - {Query: "auth", Relevant: []string{"function:Authenticate@auth.go"}}, - }, - } - data, _ := json.MarshalIndent(qc, "", " ") - os.WriteFile(dir+"/queries.json", data, 0o644) - - loaded, err := LoadQueryCorpus(dir + "/queries.json") - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 2 { - t.Fatalf("queries: got %d, want 2", len(loaded.Queries)) - } - if loaded.Queries[0].K != 5 { - t.Errorf("K: got %d, want 5", loaded.Queries[0].K) - } -} - -func TestLoadQueryCorpus_IncludesJavaCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - if loaded.Queries[5].Relevant[0] != "class:UserService@Sample.java" { - t.Fatalf("relevant: got %q, want %q", loaded.Queries[5].Relevant[0], "class:UserService@Sample.java") - } -} - -func TestLoadQueryCorpus_IncludesRustCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - if loaded.Queries[6].Relevant[0] != "function:get_user@sample.rs" { - t.Fatalf("relevant: got %q, want %q", loaded.Queries[6].Relevant[0], "function:get_user@sample.rs") - } -} - -func TestLoadQueryCorpus_IncludesJavaScriptCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - if loaded.Queries[7].Relevant[0] != "function:getUser@sample.js" { - t.Fatalf("relevant: got %q, want %q", loaded.Queries[7].Relevant[0], "function:getUser@sample.js") - } -} - -func TestLoadQueryCorpus_IncludesKotlinCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - for _, q := range loaded.Queries { - if hasRelevant(q.Relevant, "function:getUser@Sample.kt") { - return - } - } - t.Fatal("missing Kotlin relevant ID function:getUser@Sample.kt") -} - -func TestLoadQueryCorpus_IncludesPHPCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - for _, q := range loaded.Queries { - if hasRelevant(q.Relevant, "function:getUser@sample.php") { - return - } - } - t.Fatal("missing PHP relevant ID function:getUser@sample.php") -} - -func TestLoadQueryCorpus_CoversRemainingLanguages(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - - wants := []string{ - "function:get_user@sample.rb", - "function:get_user@sample.c", - "function:getUser@sample.cpp", - "function:UserService:getUser@sample.lua", - } - for _, want := range wants { - found := false - for _, q := range loaded.Queries { - if hasRelevant(q.Relevant, want) { - found = true - break - } - } - if !found { - t.Fatalf("missing relevant ID %s", want) - } - } -} - -func TestLoadQueryCorpus_UsesFixtureAlignedQueryTexts(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - - var python, typescript, rust, javascript bool - for _, q := range loaded.Queries { - switch q.Query { - case "get_user": - python = hasRelevant(q.Relevant, "function:get_user@sample.py") - case "getUser": - typescript = hasRelevant(q.Relevant, "function:getUser@sample.ts") - case "get_user Rust": - rust = hasRelevant(q.Relevant, "function:get_user@sample.rs") - case "getUser JavaScript": - javascript = hasRelevant(q.Relevant, "function:getUser@sample.js") - } - } - - if !python { - t.Fatal("missing aligned Python query case") - } - if !typescript { - t.Fatal("missing aligned TypeScript query case") - } - if !rust { - t.Fatal("missing aligned Rust query case") - } - if !javascript { - t.Fatal("missing aligned JavaScript query case") - } -} - -func TestLoadQueryCorpus_BareNameQueriesAreMultiRelevant(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - - expectations := map[string][]string{ - "UserService": { - "class:UserService@sample.go", - "class:UserService@Sample.java", - "class:UserService@sample.php", - "class:UserService@sample.cpp", - }, - "get_user": { - "function:get_user@sample.py", - "function:get_user@sample.rs", - "function:get_user@sample.rb", - "function:get_user@sample.c", - }, - "getUser": { - "function:getUser@sample.ts", - "function:getUser@sample.js", - "function:getUser@Sample.kt", - "function:getUser@sample.php", - "function:getUser@sample.cpp", - }, - } - - for query, wants := range expectations { - var got []string - for _, q := range loaded.Queries { - if q.Query == query { - got = q.Relevant - break - } - } - for _, want := range wants { - if !hasRelevant(got, want) { - t.Fatalf("query %q missing relevant ID %q (got %v)", query, want, got) - } - } - } -} - -func TestLoadQueryCorpus_IncludesNegativeControl(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - for _, q := range loaded.Queries { - if q.Query != "" && len(q.Relevant) == 0 { - return - } - } - t.Fatal("missing negative-control query with empty relevant set") -} - -func hasRelevant(relevant []string, want string) bool { - return slices.Contains(relevant, want) -} - -func TestEvaluateQueries(t *testing.T) { - cases := []QueryCase{ - { - Query: "Hello", - Relevant: []string{"a", "b"}, - K: 3, - }, - } - - mockSearch := func(query string, limit int) ([]string, error) { - return []string{"a", "c", "b"}, nil - } - - report, err := EvaluateQueries(cases, mockSearch) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if report.QueriesTotal != 1 { - t.Errorf("total: got %d, want 1", report.QueriesTotal) - } - if !approxEqual(report.AvgPAt1, 1.0) { - t.Errorf("P@1: got %f, want 1.0", report.AvgPAt1) - } - if !approxEqual(report.AvgMRR, 1.0) { - t.Errorf("MRR: got %f, want 1.0", report.AvgMRR) - } - if !approxEqual(report.AvgRecallAt5, 1.0) { - t.Errorf("R@5: got %f, want 1.0 (all relevant found within K=3)", report.AvgRecallAt5) - } -} - -func TestEvaluateQueries_Empty(t *testing.T) { - report, err := EvaluateQueries(nil, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if report.QueriesTotal != 0 { - t.Errorf("total: got %d, want 0", report.QueriesTotal) - } -} - -func TestEvaluateQueries_NegativeControlExcludedFromRankingAverages(t *testing.T) { - cases := []QueryCase{ - {Query: "positive", Relevant: []string{"a"}, K: 5}, - {Query: "negative", Relevant: []string{}, K: 5}, - } - - searchFn := func(query string, limit int) ([]string, error) { - if query == "positive" { - return []string{"a"}, nil - } - return nil, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) { - t.Fatalf("positive averages should remain perfect, got P@1=%f MRR=%f R@5=%f", report.AvgPAt1, report.AvgMRR, report.AvgRecallAt5) - } - if report.NegativeQueries != 1 || report.NegativeFalsePositives != 0 || !approxEqual(report.NegativePassRate, 1.0) { - t.Fatalf("negative control aggregation mismatch: %+v", report) - } - if report.QueriesTotal != 2 { - t.Fatalf("total queries: got %d, want 2", report.QueriesTotal) - } -} - -func TestEvaluateQueries_NegativeControlDetectsLeak(t *testing.T) { - cases := []QueryCase{{Query: "negative", Relevant: []string{}, K: 5}} - searchFn := func(query string, limit int) ([]string, error) { - return []string{"unexpected"}, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if report.NegativeQueries != 1 || report.NegativeFalsePositives != 1 || !approxEqual(report.NegativePassRate, 0.0) { - t.Fatalf("negative leak aggregation mismatch: %+v", report) - } -} - -func TestEvaluateQueries_PositiveBaselineUnchanged(t *testing.T) { - cases := []QueryCase{{Query: "Hello", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(query string, limit int) ([]string, error) { - return []string{"a", "c", "b"}, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) { - t.Fatalf("positive baseline changed: %+v", report) - } -} - -func TestEvaluateQueries_EmitsPerQueryDiagnostics(t *testing.T) { - cases := []QueryCase{ - {Query: "hit", Relevant: []string{"a", "b"}, K: 3}, - {Query: "miss", Relevant: []string{"x"}, K: 3}, - {Query: "neg", Relevant: []string{}, K: 3}, - } - searchFn := func(query string, limit int) ([]string, error) { - switch query { - case "hit": - return []string{"a", "c", "b"}, nil - case "miss": - return []string{"y", "z"}, nil - case "neg": - return []string{"unexpected"}, nil - default: - return nil, nil - } - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if len(report.PerQuery) != 3 { - t.Fatalf("PerQuery len: got %d, want 3", len(report.PerQuery)) - } - if report.PerQuery[0].Query != "hit" || !approxEqual(report.PerQuery[0].PAt1, 1.0) || !approxEqual(report.PerQuery[0].MRR, 1.0) || report.PerQuery[0].ResultsReturned != 3 || report.PerQuery[0].Kind != "positive" { - t.Fatalf("hit diagnostic mismatch: %+v", report.PerQuery[0]) - } - if report.PerQuery[1].Query != "miss" || !approxEqual(report.PerQuery[1].PAt1, 0.0) || !approxEqual(report.PerQuery[1].MRR, 0.0) || report.PerQuery[1].Kind != "positive" { - t.Fatalf("miss diagnostic mismatch: %+v", report.PerQuery[1]) - } - if report.PerQuery[2].Query != "neg" || report.PerQuery[2].Kind != "negative" || !report.PerQuery[2].FalsePositive { - t.Fatalf("neg diagnostic mismatch: %+v", report.PerQuery[2]) - } -} - -func TestEvaluateQueries_AggregatesUnchangedAfterDiagnostics(t *testing.T) { - cases := []QueryCase{{Query: "Hello", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"a", "c", "b"}, nil } - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) { - t.Fatalf("aggregates regressed: %+v", report) - } -} - -func TestEvaluateQueries_NegativeDiagnosticIncludesReturnedKeys(t *testing.T) { - cases := []QueryCase{{Query: "neg", Relevant: []string{}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"unexpected1", "unexpected2"}, nil } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if len(report.PerQuery) != 1 { - t.Fatalf("PerQuery len: got %d, want 1", len(report.PerQuery)) - } - got := report.PerQuery[0].TopResults - want := []string{"unexpected1", "unexpected2"} - if !slices.Equal(got, want) { - t.Fatalf("TopResults: got %v, want %v", got, want) - } -} - -func TestEvaluateQueries_PositiveDiagnosticIncludesReturnedKeys(t *testing.T) { - cases := []QueryCase{{Query: "hit", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"a", "c", "b"}, nil } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if len(report.PerQuery) != 1 { - t.Fatalf("PerQuery len: got %d, want 1", len(report.PerQuery)) - } - got := report.PerQuery[0].TopResults - want := []string{"a", "c", "b"} - if !slices.Equal(got, want) { - t.Fatalf("TopResults: got %v, want %v", got, want) - } -} - -func TestEvaluateQueries_DiagnosticTopResultsCappedAtFive(t *testing.T) { - cases := []QueryCase{{Query: "hit", Relevant: []string{"a"}, K: 5}} - searchFn := func(string, int) ([]string, error) { - return []string{"a", "b", "c", "d", "e", "f", "g", "h"}, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - got := report.PerQuery[0].TopResults - want := []string{"a", "b", "c", "d", "e"} - if !slices.Equal(got, want) { - t.Fatalf("TopResults cap: got %v, want %v", got, want) - } -} - -func TestEvaluateQueries_AggregatesUnchangedAfterTopResults(t *testing.T) { - cases := []QueryCase{{Query: "Hello", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"a", "c", "b"}, nil } - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) || !approxEqual(report.NegativePassRate, 0.0) { - t.Fatalf("aggregates regressed: %+v", report) - } -} diff --git a/scripts/eval.sh b/scripts/eval.sh deleted file mode 100755 index 42235a6..0000000 --- a/scripts/eval.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -CCG_BIN=${CCG_BIN:-ccg} -CCG_EVAL_CORPUS=${CCG_EVAL_CORPUS:-testdata/eval} -CCG_EVAL_NAMESPACE=${CCG_EVAL_NAMESPACE:-eval} -CCG_EVAL_DB_DRIVER=${CCG_EVAL_DB_DRIVER:-sqlite} -CCG_EVAL_DB_DSN=${CCG_EVAL_DB_DSN:-} -CCG_EVAL_KEEP_DB=${CCG_EVAL_KEEP_DB:-0} - -eval_default_db_path() { - local tmpdir - tmpdir=$(mktemp -d) - printf '%s/eval.db\n' "$tmpdir" -} - -eval_db_dsn() { - if [ -n "$CCG_EVAL_DB_DSN" ]; then - printf '%s\n' "$CCG_EVAL_DB_DSN" - return 0 - fi - eval_default_db_path -} - -eval_build_cmd() { - local db="$1" - printf '%s\n' "$CCG_BIN build $CCG_EVAL_CORPUS --db-driver $CCG_EVAL_DB_DRIVER --db-dsn $db --namespace $CCG_EVAL_NAMESPACE" -} - -eval_migrate_cmd() { - local db="$1" - printf '%s\n' "$CCG_BIN migrate --db-driver $CCG_EVAL_DB_DRIVER --db-dsn $db" -} - -eval_run_cmd() { - local db="$1" - printf '%s\n' "$CCG_BIN eval --corpus $CCG_EVAL_CORPUS --suite all --db-driver $CCG_EVAL_DB_DRIVER --db-dsn $db --namespace $CCG_EVAL_NAMESPACE" -} - -run_eval_build() { - local db="$1" - # shellcheck disable=SC2086 - $CCG_BIN build "$CCG_EVAL_CORPUS" \ - --db-driver "$CCG_EVAL_DB_DRIVER" \ - --db-dsn "$db" \ - --namespace "$CCG_EVAL_NAMESPACE" -} - -run_eval_migrate() { - local db="$1" - # shellcheck disable=SC2086 - $CCG_BIN migrate \ - --db-driver "$CCG_EVAL_DB_DRIVER" \ - --db-dsn "$db" -} - -run_eval() { - local db="$1" - # shellcheck disable=SC2086 - $CCG_BIN eval \ - --corpus "$CCG_EVAL_CORPUS" \ - --suite all \ - --db-driver "$CCG_EVAL_DB_DRIVER" \ - --db-dsn "$db" \ - --namespace "$CCG_EVAL_NAMESPACE" -} - -cleanup_eval_db() { - local db="$1" - if [ "$CCG_EVAL_KEEP_DB" = "1" ]; then - return 0 - fi - rm -f "$db" - rmdir "$(dirname "$db")" 2>/dev/null || true -} - -main() { - local db - db=$(eval_db_dsn) - trap "cleanup_eval_db '$db'" EXIT - run_eval_migrate "$db" - run_eval_build "$db" - run_eval "$db" -} - -if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then - main "$@" -fi diff --git a/scripts/eval_test.sh b/scripts/eval_test.sh deleted file mode 100755 index 8756e27..0000000 --- a/scripts/eval_test.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -ROOT_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) - -source "${ROOT_DIR}/scripts/eval.sh" - -assert_equals() { - local expected="$1" actual="$2" - [ "$expected" = "$actual" ] || { - echo "expected '$expected', got '$actual'" >&2 - exit 1 - } -} - -assert_contains() { - local haystack="$1" needle="$2" - [[ "$haystack" == *"$needle"* ]] || { - echo "expected '$haystack' to contain '$needle'" >&2 - exit 1 - } -} - -assert_fails() { - local description="$1"; shift - if ( "$@" ) >/dev/null 2>&1; then - echo "expected failure: ${description}" >&2 - exit 1 - fi -} - -reset_test_env() { - CCG_BIN="ccg" - CCG_EVAL_CORPUS="testdata/eval" - CCG_EVAL_NAMESPACE="eval" - CCG_EVAL_DB_DRIVER="sqlite" - CCG_EVAL_DB_DSN="" - CCG_EVAL_KEEP_DB=0 - TEST_TMPDIR="" -} - -test_eval_script_exists_and_is_executable() { - [ -x "${ROOT_DIR}/scripts/eval.sh" ] || { - echo "expected scripts/eval.sh to exist and be executable" >&2 - exit 1 - } -} - -test_eval_default_db_path_uses_temp_dir() { - local path - path=$(eval_default_db_path) - assert_contains "$path" "/eval.db" - [ -d "$(dirname "$path")" ] || { - echo "expected temp dir to exist for path: $path" >&2 - exit 1 - } - rm -rf "$(dirname "$path")" -} - -test_eval_build_and_run_cmds_use_shared_db_namespace_and_corpus() { - local db migrate_cmd build_cmd run_cmd - db="/tmp/eval-script-test.db" - migrate_cmd=$(eval_migrate_cmd "$db") - build_cmd=$(eval_build_cmd "$db") - run_cmd=$(eval_run_cmd "$db") - - assert_equals "ccg migrate --db-driver sqlite --db-dsn /tmp/eval-script-test.db" "$migrate_cmd" - assert_equals "ccg build testdata/eval --db-driver sqlite --db-dsn /tmp/eval-script-test.db --namespace eval" "$build_cmd" - assert_equals "ccg eval --corpus testdata/eval --suite all --db-driver sqlite --db-dsn /tmp/eval-script-test.db --namespace eval" "$run_cmd" -} - -test_eval_main_invokes_build_then_eval_in_order() { - local tmpdir stub_dir log db - tmpdir=$(mktemp -d) - stub_dir="${tmpdir}/bin" - log="${tmpdir}/ccg.log" - db="${tmpdir}/shared.db" - mkdir -p "$stub_dir" - - cat > "${stub_dir}/ccg" <> "$log" -EOF - chmod +x "${stub_dir}/ccg" - - PATH="${stub_dir}:$PATH" CCG_EVAL_DB_DSN="$db" main >/dev/null - - [ -f "$log" ] || { - echo "expected stub log to be created" >&2 - exit 1 - } - - local line1 line2 line3 - line1=$(sed -n '1p' "$log") - line2=$(sed -n '2p' "$log") - line3=$(sed -n '3p' "$log") - assert_equals "migrate --db-driver sqlite --db-dsn $db" "$line1" - assert_equals "build testdata/eval --db-driver sqlite --db-dsn $db --namespace eval" "$line2" - assert_equals "eval --corpus testdata/eval --suite all --db-driver sqlite --db-dsn $db --namespace eval" "$line3" - - rm -rf "$tmpdir" -} - -test_eval_main_supports_multitoken_ccg_bin_override() { - local tmpdir stub_dir log db - tmpdir=$(mktemp -d) - stub_dir="${tmpdir}/bin" - log="${tmpdir}/ccg.log" - db="${tmpdir}/shared.db" - mkdir -p "$stub_dir" - - cat > "${stub_dir}/ccg-stub" <> "$log" -EOF - chmod +x "${stub_dir}/ccg-stub" - - PATH="${stub_dir}:$PATH" CCG_BIN='bash ccg-stub' CCG_EVAL_DB_DSN="$db" main >/dev/null - - local line1 line2 line3 - line1=$(sed -n '1p' "$log") - line2=$(sed -n '2p' "$log") - line3=$(sed -n '3p' "$log") - assert_equals "migrate --db-driver sqlite --db-dsn $db" "$line1" - assert_equals "build testdata/eval --db-driver sqlite --db-dsn $db --namespace eval" "$line2" - assert_equals "eval --corpus testdata/eval --suite all --db-driver sqlite --db-dsn $db --namespace eval" "$line3" - - rm -rf "$tmpdir" -} - -run_test() { - local name="$1" - if ( reset_test_env; "$name" ); then - printf 'ok - %s\n' "$name" - else - printf 'not ok - %s\n' "$name" >&2 - exit 1 - fi -} - -run_test test_eval_script_exists_and_is_executable -run_test test_eval_default_db_path_uses_temp_dir -run_test test_eval_build_and_run_cmds_use_shared_db_namespace_and_corpus -run_test test_eval_main_invokes_build_then_eval_in_order -run_test test_eval_main_supports_multitoken_ccg_bin_override - -echo "eval helper tests passed" From 0c2e9a966ba7a34634389029208ac23c09065b37 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:10:36 +0900 Subject: [PATCH 02/18] ci: run go test in CI CI previously ran only vet + builds; the Go test suite was never exercised. Add a fts5-tagged test step so regressions are caught. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2381867..249da96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: - name: Run go vet run: make vet + - name: Run go test + run: CGO_ENABLED=1 go test -tags "fts5" ./... -count=1 + - name: Build release binaries run: make build From 444d0d290c7f9a6af31e488407c152cf774f69e3 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:10:36 +0900 Subject: [PATCH 03/18] Stop full-namespace scan on every retrieve_docs query The DB-scan fallback fired whenever FTS returned fewer distinct files than the wide candidate ceiling (50-500), which is nearly always, making retrieval O(namespace size) instead of O(matches). Gate the scan on the caller's actual result limit so it only supplements genuinely sparse-FTS queries, and cap the fallback load at scanRowCap to bound worst cases. Co-Authored-By: Claude Fable 5 --- internal/retrieval/retrieval.go | 4 ++++ internal/retrieval/service.go | 7 ++++++- internal/retrieval/service_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/internal/retrieval/retrieval.go b/internal/retrieval/retrieval.go index 42ff326..d83c204 100644 --- a/internal/retrieval/retrieval.go +++ b/internal/retrieval/retrieval.go @@ -36,6 +36,10 @@ type Result struct { const ( dbCandidateFloor = 50 dbCandidateCap = 500 + // scanRowCap bounds the fallback namespace scan so a sparse-FTS query cannot load an + // unbounded number of nodes+annotations. Namespaces with fewer retrievable nodes than + // this behave identically to an uncapped scan; larger ones are truncated in stable order. + scanRowCap = 5000 ) var ( diff --git a/internal/retrieval/service.go b/internal/retrieval/service.go index 853874d..3b5d44e 100644 --- a/internal/retrieval/service.go +++ b/internal/retrieval/service.go @@ -47,7 +47,11 @@ func (s *Service) FromDBWithOptions(ctx context.Context, namespace, query string candidateGroupLimit := DBCandidateLimit(limit) candidates := s.searchCandidates(ctx, query, limit) groups, nodeIDs := GroupCandidatesByFile(candidates, candidateGroupLimit) - if len(groups) < candidateGroupLimit { + // @intent supplement with a DB scan only when FTS underfills the caller's requested + // result count, not whenever it returns fewer than the wide candidate ceiling. + // @domainRule scanning the whole namespace on every query makes retrieval O(namespace); + // gating on limit keeps the scan a genuine fallback for sparse-FTS queries. + if len(groups) < limit { scanned, err := s.scanDBCandidates(ctx, effectiveNamespace, query) if err != nil { return response, err @@ -115,6 +119,7 @@ func (s *Service) scanDBCandidates(ctx context.Context, namespace, query string) Where("kind IN ?", retrievableNodeKinds). Preload("Annotation.Tags"). Order("file_path ASC, qualified_name ASC, id ASC"). + Limit(scanRowCap). Find(&nodes).Error; err != nil { return nil, fmt.Errorf("retrieve docs DB candidates: %w", err) } diff --git a/internal/retrieval/service_test.go b/internal/retrieval/service_test.go index 327d711..644818f 100644 --- a/internal/retrieval/service_test.go +++ b/internal/retrieval/service_test.go @@ -199,6 +199,34 @@ func TestServiceFromDB_supplementsPartialSearchResultsWithDBScan(t *testing.T) { } } +func TestServiceFromDB_skipsScanWhenSearchFillsLimit(t *testing.T) { + db := newRetrievalDB(t) + // FTS returns only a weak annotation-text match for a.go. + ftsNode := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Handler", Kind: model.NodeKindFunction, Name: "Handler", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, ftsNode.ID, "auth backend") + // scanOnly would score HIGHER (exact label match) but is only reachable via a DB scan; + // if the scan wrongly fires when FTS already fills the limit, this node would win and surface. + scanOnly := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Auth", Kind: model.NodeKindFunction, Name: "auth", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, scanOnly.ID, "auth service") + service := retrieval.Service{DB: db, SearchBackend: &stubSearchBackend{nodes: []model.Node{ftsNode}}} + + response, err := service.FromDB(context.Background(), "default", "auth", 1, 0, nil) + if err != nil { + t.Fatalf("FromDB returned error: %v", err) + } + if len(response.Results) != 1 { + t.Fatalf("expected one result, got %d: %+v", len(response.Results), response.Results) + } + if response.Results[0].ID != "file:a.go" { + t.Fatalf("expected FTS result only, got %q", response.Results[0].ID) + } + for _, r := range response.Results { + if r.ID == "file:b.go" { + t.Fatalf("scan-only node must not appear when FTS already fills the limit") + } + } +} + func TestServiceFromDB_limitAppliesToFileGroups(t *testing.T) { db := newRetrievalDB(t) nodeA := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.A", Kind: model.NodeKindFunction, Name: "NeedleA", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) From 8390f073ac5c0ccfb9e699c9f30baa1a0a89ee18 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:14:29 +0900 Subject: [PATCH 04/18] Rank search-engine hits above scan supplements in retrieve_docs FTS/ts_rank candidate order was discarded and results were sorted purely by the hand-tuned annotation score, so a high-scoring scan supplement could bury a genuine search-engine hit. Capture each file's search-engine rank before the scan merge and sort engine-first: backend hits keep their relevance order and outrank scan-only supplements, with the annotation score as the refining tie-break. Co-Authored-By: Claude Fable 5 --- internal/retrieval/service.go | 42 +++++++++++++++++++++++++++--- internal/retrieval/service_test.go | 25 ++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/internal/retrieval/service.go b/internal/retrieval/service.go index 3b5d44e..75d46d4 100644 --- a/internal/retrieval/service.go +++ b/internal/retrieval/service.go @@ -46,6 +46,9 @@ func (s *Service) FromDBWithOptions(ctx context.Context, namespace, query string candidateGroupLimit := DBCandidateLimit(limit) candidates := s.searchCandidates(ctx, query, limit) + // Capture the search engine's per-file rank order before the scan supplement is merged in, + // so engine hits keep their relevance ordering and outrank scan-only supplements. + ftsRanks := ftsFileRanks(candidates) groups, nodeIDs := GroupCandidatesByFile(candidates, candidateGroupLimit) // @intent supplement with a DB scan only when FTS underfills the caller's requested // result count, not whenever it returns fewer than the wide candidate ceiling. @@ -74,7 +77,7 @@ func (s *Service) FromDBWithOptions(ctx context.Context, namespace, query string result.Matches = DBMatches(group.Nodes, annotations) response.Results = append(response.Results, result) } - sortRetrieveResults(response.Results) + sortRetrieveResults(response.Results, ftsRanks) if len(response.Results) > limit { response.Results = response.Results[:limit] } @@ -164,9 +167,42 @@ func mergeCandidates(primary, supplemental []model.Node) []model.Node { return merged } -// @intent keep DB retrieve ordering score-first after all candidates have been structurally scored. -func sortRetrieveResults(results []Result) { +// ftsFileRanks records each file's best (earliest) position in the search engine's +// rank-ordered candidate list, keyed by file path. +// @intent preserve the search backend's relevance ordering as an authoritative ranking signal. +func ftsFileRanks(candidates []model.Node) map[string]int { + ranks := make(map[string]int, len(candidates)) + for pos, node := range candidates { + if !IsRetrievableNodeKind(node.Kind) { + continue + } + filePath := strings.TrimSpace(node.FilePath) + if filePath == "" { + continue + } + if _, seen := ranks[filePath]; !seen { + ranks[filePath] = pos + } + } + return ranks +} + +// @intent order DB retrieve results engine-first: search-backend hits keep their relevance rank +// and outrank scan-only supplements, with the structured annotation score as the refining signal. +func sortRetrieveResults(results []Result, ftsRanks map[string]int) { + rankOf := func(r Result) (int, bool) { + pos, ok := ftsRanks[strings.TrimPrefix(r.ID, "file:")] + return pos, ok + } sort.SliceStable(results, func(i, j int) bool { + ri, iHit := rankOf(results[i]) + rj, jHit := rankOf(results[j]) + if iHit != jHit { + return iHit // engine hits before scan-only supplements + } + if iHit && ri != rj { + return ri < rj // earlier engine rank wins + } if results[i].Score != results[j].Score { return results[i].Score > results[j].Score } diff --git a/internal/retrieval/service_test.go b/internal/retrieval/service_test.go index 644818f..2496df8 100644 --- a/internal/retrieval/service_test.go +++ b/internal/retrieval/service_test.go @@ -227,6 +227,31 @@ func TestServiceFromDB_skipsScanWhenSearchFillsLimit(t *testing.T) { } } +func TestServiceFromDB_ftsHitOutranksHigherScoredScanSupplement(t *testing.T) { + db := newRetrievalDB(t) + // FTS returns a.go with only a weak annotation-text match. + weakFTS := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Handler", Kind: model.NodeKindFunction, Name: "Handler", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, weakFTS.ID, "auth request") + // b.go is reachable only via the scan supplement and scores much higher (exact label + high-weight tags). + strongScan := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Auth", Kind: model.NodeKindFunction, Name: "auth", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, strongScan.ID, "auth", model.DocTag{Kind: model.TagIntent, Value: "auth"}, model.DocTag{Kind: model.TagDomainRule, Value: "auth"}) + service := retrieval.Service{DB: db, SearchBackend: &stubSearchBackend{nodes: []model.Node{weakFTS}}} + + response, err := service.FromDB(context.Background(), "default", "auth", 2, 0, nil) + if err != nil { + t.Fatalf("FromDB returned error: %v", err) + } + if len(response.Results) != 2 { + t.Fatalf("expected FTS hit plus scan supplement, got %d: %+v", len(response.Results), response.Results) + } + if response.Results[0].ID != "file:a.go" { + t.Fatalf("search-engine hit must rank above a higher-scored scan supplement, got %q first", response.Results[0].ID) + } + if response.Results[1].ID != "file:b.go" { + t.Fatalf("expected scan supplement second, got %q", response.Results[1].ID) + } +} + func TestServiceFromDB_limitAppliesToFileGroups(t *testing.T) { db := newRetrievalDB(t) nodeA := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.A", Kind: model.NodeKindFunction, Name: "NeedleA", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) From dcf4c54a9f09da27cdccd1df3cd8c12f8b3fc6b3 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:20:42 +0900 Subject: [PATCH 05/18] Add pg_trgm fuzzy symbol fallback to Postgres search Exact tsquery cannot match misspelled queries. When tsquery underfills the requested limit, supplement with pg_trgm trigram similarity on nodes.name / nodes.qualified_name so typos still return candidates. The extension and its GIN indexes are created best-effort in Migrate: if pg_trgm is unavailable (insufficient privilege) it logs a warning and search degrades to exact-only. The fuzzy query swallows errors so a missing extension never breaks exact search. Behaviorally validated only against a live Postgres via the postgres build tag (TEST_POSTGRES_DSN); not exercised by the default/CI suite. Co-Authored-By: Claude Fable 5 --- internal/store/search/postgres.go | 92 +++++++++++++++++++++++++- internal/store/search/postgres_test.go | 29 ++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/internal/store/search/postgres.go b/internal/store/search/postgres.go index 3b0404c..7a124cf 100644 --- a/internal/store/search/postgres.go +++ b/internal/store/search/postgres.go @@ -4,6 +4,7 @@ package search import ( "context" "fmt" + "log/slog" "gorm.io/gorm" @@ -13,6 +14,11 @@ import ( "github.com/tae2089/code-context-graph/internal/model" ) +// fuzzyWordSimilarityThreshold is the minimum pg_trgm word_similarity for a symbol name to be +// accepted as a typo-tolerant fuzzy match. Tuned to admit single-character typos while rejecting +// unrelated names. +const fuzzyWordSimilarityThreshold = 0.4 + // PostgresBackend is a full-text search backend based on PostgreSQL tsvector. // @intent Handles full-text search indexing and querying in a PostgreSQL environment. type PostgresBackend struct{} @@ -74,6 +80,20 @@ func (p *PostgresBackend) Migrate(db *gorm.DB) error { return trace.Wrap(err, "create gin index") } + // pg_trgm powers typo-tolerant fuzzy symbol matching. The extension may need + // elevated privileges, so treat its absence as a graceful downgrade to exact FTS + // rather than a fatal migration error. + if err := db.Exec(`CREATE EXTENSION IF NOT EXISTS pg_trgm`).Error; err != nil { + slog.Warn("pg_trgm unavailable; fuzzy symbol search disabled", trace.SlogError(err)) + return nil + } + if err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_name_trgm ON nodes USING gin (name gin_trgm_ops)`).Error; err != nil { + return trace.Wrap(err, "create name trgm index") + } + if err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_qualified_name_trgm ON nodes USING gin (qualified_name gin_trgm_ops)`).Error; err != nil { + return trace.Wrap(err, "create qualified_name trgm index") + } + return nil } @@ -154,8 +174,10 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string, return nil, trace.Wrap(err, "ts_query") } + // A zero-hit tsquery is not terminal: a misspelled query legitimately matches nothing + // exactly, and the pg_trgm fuzzy supplement below can still surface candidates. if len(rows) == 0 { - return nil, nil + return p.appendFuzzyMatches(ctx, db, query, ns, nil, limit), nil } nodeIDs := make([]uint, len(rows)) @@ -187,8 +209,76 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string, } } + // When exact FTS underfills the requested limit, supplement with typo-tolerant + // trigram matches on symbol names so misspelled queries still return candidates. + if len(result) < limit { + result = p.appendFuzzyMatches(ctx, db, query, ns, result, limit) + } + result = promoteExactNameMatch(result, query) return result, nil } +// appendFuzzyMatches supplements exact-FTS results with pg_trgm similarity matches on +// symbol names, ordered by similarity. It is best-effort: when pg_trgm is unavailable the +// query errors and the exact results are returned unchanged. +// @intent add typo tolerance to Postgres search without letting a missing extension break exact search. +func (p *PostgresBackend) appendFuzzyMatches(ctx context.Context, db *gorm.DB, query, ns string, existing []model.Node, limit int) []model.Node { + remaining := limit - len(existing) + if remaining <= 0 { + return existing + } + seen := make(map[uint]struct{}, len(existing)) + for _, n := range existing { + seen[n.ID] = struct{}{} + } + + // word_similarity (not similarity) is used so a short query matches the best-fitting + // extent of a longer symbol name; plain similarity penalizes length differences and + // misses typos like "authentcate" inside "AuthenticateUser". + var rows []resultRow + fuzzySQL := ` + SELECT id AS node_id + FROM nodes + WHERE namespace = ? + AND (word_similarity(?, name) >= ? OR word_similarity(?, qualified_name) >= ?) + ORDER BY GREATEST(word_similarity(?, name), word_similarity(?, qualified_name)) DESC + LIMIT ?` + if err := db.WithContext(ctx).Raw(fuzzySQL, ns, query, fuzzyWordSimilarityThreshold, query, fuzzyWordSimilarityThreshold, query, query, remaining+len(existing)).Scan(&rows).Error; err != nil { + slog.Debug("pg_trgm fuzzy supplement skipped", trace.SlogError(err)) + return existing + } + + fuzzyIDs := make([]uint, 0, len(rows)) + for _, r := range rows { + if _, ok := seen[r.NodeID]; ok { + continue + } + seen[r.NodeID] = struct{}{} + fuzzyIDs = append(fuzzyIDs, r.NodeID) + } + if len(fuzzyIDs) == 0 { + return existing + } + + var nodes []model.Node + if err := db.WithContext(ctx).Where("id IN ?", fuzzyIDs).Where("namespace = ?", ns).Find(&nodes).Error; err != nil { + slog.Debug("pg_trgm fuzzy node load skipped", trace.SlogError(err)) + return existing + } + byID := make(map[uint]model.Node, len(nodes)) + for _, n := range nodes { + byID[n.ID] = n + } + for _, id := range fuzzyIDs { + if n, ok := byID[id]; ok && n.ID != 0 { + existing = append(existing, n) + if len(existing) >= limit { + break + } + } + } + return existing +} + var _ Backend = (*PostgresBackend)(nil) diff --git a/internal/store/search/postgres_test.go b/internal/store/search/postgres_test.go index 6caa22a..faca472 100644 --- a/internal/store/search/postgres_test.go +++ b/internal/store/search/postgres_test.go @@ -275,6 +275,35 @@ func TestPostgresFTS_Query(t *testing.T) { } } +func TestPostgresFTS_Query_FuzzyTypoFallsBackToTrigram(t *testing.T) { + db := setupPostgresDB(t) + seedPostgresNodes(t, db) + + backend := NewPostgresBackend() + if err := backend.Migrate(db); err != nil { + t.Fatal(err) + } + if err := backend.Rebuild(context.Background(), db); err != nil { + t.Fatal(err) + } + + // "authentcate" is a typo (missing 'i') that exact tsquery cannot match; + // the pg_trgm supplement should still surface AuthenticateUser by symbol-name similarity. + nodes, err := backend.Query(context.Background(), db, "authentcate", 10) + if err != nil { + t.Fatal(err) + } + found := false + for _, n := range nodes { + if n.QualifiedName == "pkg.AuthenticateUser" { + found = true + } + } + if !found { + t.Errorf("expected fuzzy trigram match to surface pkg.AuthenticateUser for typo query, got %+v", nodes) + } +} + func TestPostgresFTS_Query_NamespaceIsolation(t *testing.T) { db := setupPostgresDB(t) backend := NewPostgresBackend() From a610422cc52b4f986de6c4d442fa03fb968a570e Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:40:09 +0900 Subject: [PATCH 06/18] Extract shared path-safety mechanics into internal/safepath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The symlink-walk, canonicalization, and Rel-based containment mechanics were duplicated across namespacefs and the docs/analysis/parse MCP handlers. Extract the three genuinely-shared primitives — EnsureNoSymlinkInPath, Canonical(path, allowMissingLeaf) (merging the former canonicalPath/canonicalExistingPath pair), and IsWithinRoot — into a single internal/safepath package with direct unit tests, closing a gap where namespacefs had none. The three distinct containment *policies* (prefix-reject-symlink in docs, Rel-follow-symlink in analysis, walk-reject-symlink in namespacefs) and all error strings stay at their call sites: merging them would change security behavior. Structural change only; all handler path-traversal and symlink-escape tests unchanged and green. Co-Authored-By: Claude Fable 5 --- internal/mcp/handler_analysis.go | 44 +---------- internal/mcp/handler_parse.go | 31 +------- internal/namespacefs/workspace.go | 26 +----- internal/safepath/safepath.go | 94 ++++++++++++++++++++++ internal/safepath/safepath_test.go | 123 +++++++++++++++++++++++++++++ 5 files changed, 226 insertions(+), 92 deletions(-) create mode 100644 internal/safepath/safepath.go create mode 100644 internal/safepath/safepath_test.go diff --git a/internal/mcp/handler_analysis.go b/internal/mcp/handler_analysis.go index 787fadc..71cfed4 100644 --- a/internal/mcp/handler_analysis.go +++ b/internal/mcp/handler_analysis.go @@ -4,7 +4,6 @@ package mcp import ( "context" "fmt" - "path/filepath" "slices" "github.com/mark3labs/mcp-go/mcp" @@ -19,6 +18,7 @@ import ( "github.com/tae2089/code-context-graph/internal/model" "github.com/tae2089/code-context-graph/internal/obs" "github.com/tae2089/code-context-graph/internal/paging" + "github.com/tae2089/code-context-graph/internal/safepath" ) const ( @@ -614,7 +614,7 @@ func validateRepoRootWithin(repoRoot, configuredRepoRoot, namespaceRoot string) if len(allowedRoots) == 0 { return "", fmt.Errorf("analysis repo root is not configured") } - repo, err := canonicalPath(repoRoot) + repo, err := safepath.Canonical(repoRoot, false) if err != nil { return "", fmt.Errorf("invalid repo_root: %w", err) } @@ -653,11 +653,11 @@ func sliceContainsString(values []string, target string) bool { // @requires allowedRoots must be non-empty and each entry must be a valid filesystem path. func validatePathWithinAllowedRoots(target string, allowedRoots []string) (bool, error) { for _, root := range allowedRoots { - base, err := canonicalPath(root) + base, err := safepath.Canonical(root, false) if err != nil { return false, err } - within, err := isWithinRoot(base, target) + within, err := safepath.IsWithinRoot(base, target) if err != nil { return false, err } @@ -667,39 +667,3 @@ func validatePathWithinAllowedRoots(target string, allowedRoots []string) (bool, } return false, nil } - -// isWithinRoot reports whether target is the same as root or a descendant of it. -// @intent detect path traversal by checking the relative path does not escape upward. -func isWithinRoot(root, target string) (bool, error) { - rel, err := filepath.Rel(root, target) - if err != nil { - return false, err - } - if rel == "." { - return true, nil - } - if rel == ".." { - return false, nil - } - if len(rel) >= 3 && rel[:3] == ".."+string(filepath.Separator) { - return false, nil - } - if filepath.IsAbs(rel) { - return false, nil - } - return true, nil -} - -// canonicalPath resolves path to an absolute, symlink-free, cleaned filesystem path. -// @intent normalize user-supplied paths before comparison to prevent symlink-based boundary escapes. -func canonicalPath(path string) (string, error) { - abs, err := filepath.Abs(path) - if err != nil { - return "", err - } - real, err := filepath.EvalSymlinks(abs) - if err != nil { - return "", err - } - return filepath.Clean(real), nil -} diff --git a/internal/mcp/handler_parse.go b/internal/mcp/handler_parse.go index 76b07c4..7edc214 100644 --- a/internal/mcp/handler_parse.go +++ b/internal/mcp/handler_parse.go @@ -3,10 +3,7 @@ package mcp import ( "context" - "errors" "fmt" - "os" - "path/filepath" "slices" "time" @@ -17,6 +14,7 @@ import ( flowspkg "github.com/tae2089/code-context-graph/internal/analysis/flows" "github.com/tae2089/code-context-graph/internal/obs" postprocesspolicy "github.com/tae2089/code-context-graph/internal/postprocess/policy" + "github.com/tae2089/code-context-graph/internal/safepath" "github.com/tae2089/code-context-graph/internal/service" ) @@ -522,7 +520,7 @@ func (h *handlers) validateAnalysisPath(path string) (string, error) { if len(allowedRoots) == 0 { return "", fmt.Errorf("analysis root is not configured") } - target, err := canonicalExistingPath(path) + target, err := safepath.Canonical(path, true) if err != nil { return "", fmt.Errorf("invalid path: %w", err) } @@ -535,28 +533,3 @@ func (h *handlers) validateAnalysisPath(path string) (string, error) { } return target, nil } - -// @intent resolve a path to its absolute, symlink-evaluated form for analysis-root containment checks. -// @domainRule symlink evaluation must happen before containment checks against analysis roots. -// @ensures returned path is cleaned and absolute; existing paths are fully symlink-resolved. -func canonicalExistingPath(path string) (string, error) { - abs, err := filepath.Abs(path) - if err != nil { - return "", err - } - clean := filepath.Clean(abs) - real, err := filepath.EvalSymlinks(clean) - if err == nil { - return filepath.Clean(real), nil - } - if !errors.Is(err, os.ErrNotExist) { - return "", err - } - parent := filepath.Dir(clean) - base := filepath.Base(clean) - parentReal, parentErr := filepath.EvalSymlinks(parent) - if parentErr != nil { - return "", err - } - return filepath.Join(parentReal, base), nil -} diff --git a/internal/namespacefs/workspace.go b/internal/namespacefs/workspace.go index fa5667e..cb91e29 100644 --- a/internal/namespacefs/workspace.go +++ b/internal/namespacefs/workspace.go @@ -11,6 +11,8 @@ import ( "path/filepath" "slices" "strings" + + "github.com/tae2089/code-context-graph/internal/safepath" ) // Default upload size limits applied when callers do not provide their own Limits. @@ -133,29 +135,7 @@ func ValidatePath(namespace, filePath string) error { // @intent prevent symlink traversal from escaping the namespace root before any filesystem mutation. // @param allowMissingLeaf when true, returns the joined path even when the leaf does not yet exist. func EnsureNoSymlinkInPath(root, relPath string, allowMissingLeaf bool) (string, error) { - cleanRel := filepath.Clean(relPath) - if cleanRel == "." { - return root, nil - } - current := root - segments := strings.Split(cleanRel, string(filepath.Separator)) - for i, segment := range segments { - current = filepath.Join(current, segment) - info, err := os.Lstat(current) - if err != nil { - if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) && i == len(segments)-1 { - return current, nil - } - if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) { - continue - } - return "", err - } - if info.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("symlink paths are not allowed") - } - } - return current, nil + return safepath.EnsureNoSymlinkInPath(root, relPath, allowMissingLeaf) } // SafeWrite atomically writes data to path using a temp file and rename, refusing to follow symlinks. diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go new file mode 100644 index 0000000..3c67173 --- /dev/null +++ b/internal/safepath/safepath.go @@ -0,0 +1,94 @@ +// @index safepath는 여러 핸들러가 공유하는 저수준 경로 안전 프리미티브를 단일화한다. +// 정책(prefix/Rel/walk 기반 containment)과 에러 문자열은 호출부에 남기고, 여기서는 +// 공통 메커니즘(심링크 거부 walk, canonical 정규화, Rel 기반 containment)만 제공한다. +package safepath + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// EnsureNoSymlinkInPath walks each path segment from root to relPath rejecting symlinks. +// @intent prevent symlink traversal from escaping a trusted root before any filesystem mutation. +// @param allowMissingLeaf when true, returns the joined path even when the leaf does not yet exist. +func EnsureNoSymlinkInPath(root, relPath string, allowMissingLeaf bool) (string, error) { + cleanRel := filepath.Clean(relPath) + if cleanRel == "." { + return root, nil + } + current := root + segments := strings.Split(cleanRel, string(filepath.Separator)) + for i, segment := range segments { + current = filepath.Join(current, segment) + info, err := os.Lstat(current) + if err != nil { + if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) && i == len(segments)-1 { + return current, nil + } + if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) { + continue + } + return "", err + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("symlink paths are not allowed") + } + } + return current, nil +} + +// Canonical resolves path to an absolute, symlink-free, cleaned filesystem path. +// @intent normalize user-supplied paths before containment comparison to prevent symlink-based escapes. +// @param allowMissingLeaf when true, a non-existent leaf is tolerated by resolving the parent and +// appending the unresolved base; when false, a missing path is an error (strict existence). +func Canonical(path string, allowMissingLeaf bool) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + clean := filepath.Clean(abs) + real, err := filepath.EvalSymlinks(clean) + if err == nil { + return filepath.Clean(real), nil + } + if !allowMissingLeaf { + return "", err + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + parent := filepath.Dir(clean) + base := filepath.Base(clean) + parentReal, parentErr := filepath.EvalSymlinks(parent) + if parentErr != nil { + return "", err + } + return filepath.Join(parentReal, base), nil +} + +// IsWithinRoot reports whether target is the same as root or a descendant of it. +// @intent detect path traversal by checking the relative path does not escape upward. +// @requires root and target should already be canonicalized (see Canonical) for symlink-safe comparison. +func IsWithinRoot(root, target string) (bool, error) { + rel, err := filepath.Rel(root, target) + if err != nil { + return false, err + } + if rel == "." { + return true, nil + } + if rel == ".." { + return false, nil + } + if len(rel) >= 3 && rel[:3] == ".."+string(filepath.Separator) { + return false, nil + } + if filepath.IsAbs(rel) { + return false, nil + } + return true, nil +} diff --git a/internal/safepath/safepath_test.go b/internal/safepath/safepath_test.go new file mode 100644 index 0000000..1ecfbf5 --- /dev/null +++ b/internal/safepath/safepath_test.go @@ -0,0 +1,123 @@ +package safepath_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/tae2089/code-context-graph/internal/safepath" +) + +func TestEnsureNoSymlinkInPath_AllowsPlainNestedPath(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "a", "b"), 0o755); err != nil { + t.Fatal(err) + } + got, err := safepath.EnsureNoSymlinkInPath(root, filepath.Join("a", "b"), false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != filepath.Join(root, "a", "b") { + t.Fatalf("unexpected path %q", got) + } +} + +func TestEnsureNoSymlinkInPath_DotReturnsRoot(t *testing.T) { + root := t.TempDir() + got, err := safepath.EnsureNoSymlinkInPath(root, ".", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != root { + t.Fatalf("expected root, got %q", got) + } +} + +func TestEnsureNoSymlinkInPath_RejectsSymlinkSegment(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink semantics differ on windows") + } + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outside, "secret"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + _, err := safepath.EnsureNoSymlinkInPath(root, filepath.Join("link", "secret"), false) + if err == nil { + t.Fatal("expected symlink segment to be rejected") + } +} + +func TestEnsureNoSymlinkInPath_MissingLeafToleratedWhenAllowed(t *testing.T) { + root := t.TempDir() + rel := filepath.Join("not", "there.txt") + got, err := safepath.EnsureNoSymlinkInPath(root, rel, true) + if err != nil { + t.Fatalf("expected missing leaf tolerated, got %v", err) + } + if got != filepath.Join(root, rel) { + t.Fatalf("unexpected path %q", got) + } +} + +func TestEnsureNoSymlinkInPath_MissingLeafErrorsWhenNotAllowed(t *testing.T) { + root := t.TempDir() + if _, err := safepath.EnsureNoSymlinkInPath(root, "missing.txt", false); err == nil { + t.Fatal("expected error for missing leaf when allowMissingLeaf is false") + } +} + +func TestCanonical_StrictRequiresExistence(t *testing.T) { + root := t.TempDir() + missing := filepath.Join(root, "nope") + if _, err := safepath.Canonical(missing, false); err == nil { + t.Fatal("expected strict Canonical to error on missing path") + } + got, err := safepath.Canonical(root, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + real, _ := filepath.EvalSymlinks(root) + if got != real { + t.Fatalf("expected canonical %q, got %q", real, got) + } +} + +func TestCanonical_TolerantResolvesParentForMissingLeaf(t *testing.T) { + root := t.TempDir() + real, _ := filepath.EvalSymlinks(root) + got, err := safepath.Canonical(filepath.Join(root, "newleaf"), true) + if err != nil { + t.Fatalf("expected tolerant Canonical to succeed, got %v", err) + } + if got != filepath.Join(real, "newleaf") { + t.Fatalf("unexpected path %q", got) + } +} + +func TestIsWithinRoot(t *testing.T) { + root := "/srv/data" + cases := []struct { + target string + want bool + }{ + {"/srv/data", true}, + {"/srv/data/sub/file", true}, + {"/srv/dataother", false}, + {"/srv", false}, + {"/etc/passwd", false}, + } + for _, c := range cases { + got, err := safepath.IsWithinRoot(root, c.target) + if err != nil { + t.Fatalf("IsWithinRoot(%q,%q) error: %v", root, c.target, err) + } + if got != c.want { + t.Errorf("IsWithinRoot(%q,%q)=%v want %v", root, c.target, got, c.want) + } + } +} From 61a01fe4ac3c2e6d6433d7cee391450be5aebbd6 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:53:33 +0900 Subject: [PATCH 07/18] Make pg_trgm fuzzy indexable, precise, and corpus-scoped Address code-review findings on the fuzzy fallback: - Use the index-accelerated `<%` operator (rewritten to `%>`, which gin_trgm_ops supports) instead of `word_similarity(?, col) >= const`, which can never use the trigram indexes. Confirmed via EXPLAIN that the operator form yields a Bitmap Index Scan. - Fire fuzzy only on a total exact-FTS miss, not whenever exact underfills the limit, so precise queries are no longer diluted with loose matches. - Scope fuzzy to nodes that have a search_documents row (JOIN), keeping it within the same corpus and kind mix as the exact path. - Set the word_similarity threshold via transaction-local set_config so the operator cutoff matches the tuned value; log real failures at Warn while still degrading gracefully, and do not log on context cancellation. Co-Authored-By: Claude Fable 5 --- internal/store/search/postgres.go | 99 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/internal/store/search/postgres.go b/internal/store/search/postgres.go index 7a124cf..b0c9a24 100644 --- a/internal/store/search/postgres.go +++ b/internal/store/search/postgres.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "log/slog" + "strconv" "gorm.io/gorm" @@ -175,9 +176,11 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string, } // A zero-hit tsquery is not terminal: a misspelled query legitimately matches nothing - // exactly, and the pg_trgm fuzzy supplement below can still surface candidates. + // exactly, and the pg_trgm fuzzy supplement can still surface candidates. Fuzzy fires + // only on a total exact miss so it never dilutes the precision of queries that did match. if len(rows) == 0 { - return p.appendFuzzyMatches(ctx, db, query, ns, nil, limit), nil + fuzzy := p.appendFuzzyMatches(ctx, db, query, ns, limit) + return promoteExactNameMatch(fuzzy, query), nil } nodeIDs := make([]uint, len(rows)) @@ -209,76 +212,74 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string, } } - // When exact FTS underfills the requested limit, supplement with typo-tolerant - // trigram matches on symbol names so misspelled queries still return candidates. - if len(result) < limit { - result = p.appendFuzzyMatches(ctx, db, query, ns, result, limit) - } - result = promoteExactNameMatch(result, query) return result, nil } -// appendFuzzyMatches supplements exact-FTS results with pg_trgm similarity matches on -// symbol names, ordered by similarity. It is best-effort: when pg_trgm is unavailable the -// query errors and the exact results are returned unchanged. +// appendFuzzyMatches returns up to limit pg_trgm fuzzy matches on symbol names for queries +// that produced no exact FTS hit (typo tolerance). It is best-effort: if pg_trgm is +// unavailable the query errors and an empty result is returned so exact search still works. // @intent add typo tolerance to Postgres search without letting a missing extension break exact search. -func (p *PostgresBackend) appendFuzzyMatches(ctx context.Context, db *gorm.DB, query, ns string, existing []model.Node, limit int) []model.Node { - remaining := limit - len(existing) - if remaining <= 0 { - return existing - } - seen := make(map[uint]struct{}, len(existing)) - for _, n := range existing { - seen[n.ID] = struct{}{} +// @domainRule only nodes that also have a search_documents row are eligible, so fuzzy stays +// within the same corpus and kind mix as the exact FTS path. +func (p *PostgresBackend) appendFuzzyMatches(ctx context.Context, db *gorm.DB, query, ns string, limit int) []model.Node { + if limit <= 0 { + return nil } - // word_similarity (not similarity) is used so a short query matches the best-fitting - // extent of a longer symbol name; plain similarity penalizes length differences and - // misses typos like "authentcate" inside "AuthenticateUser". + // The `<%` operator is index-accelerated by the gin_trgm_ops indexes on name/qualified_name + // (a functional `word_similarity(...) >= const` filter cannot use them); its cutoff is the + // session-local word_similarity_threshold, set here so the operator and our tuned threshold + // agree. Ordering uses the functional form over the already-filtered small set. var rows []resultRow fuzzySQL := ` - SELECT id AS node_id - FROM nodes - WHERE namespace = ? - AND (word_similarity(?, name) >= ? OR word_similarity(?, qualified_name) >= ?) - ORDER BY GREATEST(word_similarity(?, name), word_similarity(?, qualified_name)) DESC + SELECT n.id AS node_id + FROM nodes n + JOIN search_documents sd ON sd.node_id = n.id AND sd.namespace = n.namespace + WHERE n.namespace = ? + AND (? <% n.name OR ? <% n.qualified_name) + ORDER BY GREATEST(word_similarity(?, n.name), word_similarity(?, n.qualified_name)) DESC LIMIT ?` - if err := db.WithContext(ctx).Raw(fuzzySQL, ns, query, fuzzyWordSimilarityThreshold, query, fuzzyWordSimilarityThreshold, query, query, remaining+len(existing)).Scan(&rows).Error; err != nil { - slog.Debug("pg_trgm fuzzy supplement skipped", trace.SlogError(err)) - return existing - } - - fuzzyIDs := make([]uint, 0, len(rows)) - for _, r := range rows { - if _, ok := seen[r.NodeID]; ok { - continue + err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if e := tx.Exec(`SELECT set_config('pg_trgm.word_similarity_threshold', ?, true)`, + strconv.FormatFloat(fuzzyWordSimilarityThreshold, 'f', -1, 64)).Error; e != nil { + return e } - seen[r.NodeID] = struct{}{} - fuzzyIDs = append(fuzzyIDs, r.NodeID) + return tx.Raw(fuzzySQL, ns, query, query, query, query, limit).Scan(&rows).Error + }) + if err != nil { + // Context cancellation is the caller's concern, not a fuzzy fault; only surface real errors. + if ctx.Err() == nil { + slog.Warn("pg_trgm fuzzy supplement failed", trace.SlogError(err)) + } + return nil } - if len(fuzzyIDs) == 0 { - return existing + if len(rows) == 0 { + return nil } + nodeIDs := make([]uint, len(rows)) + for i, r := range rows { + nodeIDs[i] = r.NodeID + } var nodes []model.Node - if err := db.WithContext(ctx).Where("id IN ?", fuzzyIDs).Where("namespace = ?", ns).Find(&nodes).Error; err != nil { - slog.Debug("pg_trgm fuzzy node load skipped", trace.SlogError(err)) - return existing + if err := db.WithContext(ctx).Where("id IN ?", nodeIDs).Where("namespace = ?", ns).Find(&nodes).Error; err != nil { + if ctx.Err() == nil { + slog.Warn("pg_trgm fuzzy node load failed", trace.SlogError(err)) + } + return nil } byID := make(map[uint]model.Node, len(nodes)) for _, n := range nodes { byID[n.ID] = n } - for _, id := range fuzzyIDs { + ordered := make([]model.Node, 0, len(nodeIDs)) + for _, id := range nodeIDs { if n, ok := byID[id]; ok && n.ID != 0 { - existing = append(existing, n) - if len(existing) >= limit { - break - } + ordered = append(ordered, n) } } - return existing + return ordered } var _ Backend = (*PostgresBackend)(nil) From 72bee1e64bb241fc9eecee75c5ab3ed2d6e7b008 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 21:53:33 +0900 Subject: [PATCH 08/18] Log fallback-scan truncation and drop dead blank-assignment block - scanDBCandidates now warns when the fallback scan hits scanRowCap, so a silently truncated result set is observable instead of looking complete. - Remove the pointless `var ( _ = dbCandidateFloor; _ = dbCandidateCap )` block: both constants are used by DBCandidateLimit. Co-Authored-By: Claude Fable 5 --- internal/retrieval/retrieval.go | 5 ----- internal/retrieval/service.go | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/retrieval/retrieval.go b/internal/retrieval/retrieval.go index d83c204..50d7405 100644 --- a/internal/retrieval/retrieval.go +++ b/internal/retrieval/retrieval.go @@ -41,8 +41,3 @@ const ( // this behave identically to an uncapped scan; larger ones are truncated in stable order. scanRowCap = 5000 ) - -var ( - _ = dbCandidateFloor - _ = dbCandidateCap -) diff --git a/internal/retrieval/service.go b/internal/retrieval/service.go index 75d46d4..eea7010 100644 --- a/internal/retrieval/service.go +++ b/internal/retrieval/service.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "sort" "strings" @@ -126,6 +127,11 @@ func (s *Service) scanDBCandidates(ctx context.Context, namespace, query string) Find(&nodes).Error; err != nil { return nil, fmt.Errorf("retrieve docs DB candidates: %w", err) } + if len(nodes) == scanRowCap { + // The scan hit its ceiling: matching nodes sorting after the cap are not considered. + slog.WarnContext(ctx, "retrieve_docs fallback scan truncated at cap; some matches may be omitted", + "namespace", namespace, "cap", scanRowCap) + } if len(nodes) == 0 { return nil, nil } From 448485d21b4e5db6dd28d9f8ab2d39c34a20d3dd Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:13 +0900 Subject: [PATCH 09/18] Block git option injection via base ref and handle non-ASCII paths The caller-supplied base ref of detect_changes/get_affected_flows sat before "--" with no validation, so values like --output= were parsed as git diff options. Reject refs starting with '-' and add the "--" separator to ChangedFiles. Also run diff with core.quotePath=false: quoted octal output for non-ASCII paths broke hunk-to-file attribution and file matching. Co-Authored-By: Claude Fable 5 --- internal/analysis/changes/git.go | 23 +++++++++++++-- internal/analysis/changes/git_test.go | 40 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/internal/analysis/changes/git.go b/internal/analysis/changes/git.go index 6006085..e4d1b11 100644 --- a/internal/analysis/changes/git.go +++ b/internal/analysis/changes/git.go @@ -37,7 +37,10 @@ func NewExecGitClient() *ExecGitClient { // @requires repoDir points to a valid git working tree // @ensures returned file paths are trimmed and exclude blank lines func (g *ExecGitClient) ChangedFiles(ctx context.Context, repoDir, baseRef string) ([]string, error) { - out, err := runGitLimited(ctx, repoDir, []string{"diff", "--name-only", baseRef}) + if err := validateBaseRef(baseRef); err != nil { + return nil, err + } + out, err := runGitLimited(ctx, repoDir, []string{"-c", "core.quotePath=false", "diff", "--name-only", baseRef, "--"}) if err != nil { return nil, trace.Wrap(err, "git diff --name-only") } @@ -66,7 +69,10 @@ func (g *ExecGitClient) ChangedFiles(ctx context.Context, repoDir, baseRef strin // @requires repoDir points to a valid git working tree // @ensures each returned hunk has a file path and inclusive start/end lines func (g *ExecGitClient) DiffHunks(ctx context.Context, repoDir, baseRef string, paths []string) ([]Hunk, error) { - args := []string{"diff", "-U0", baseRef, "--"} + if err := validateBaseRef(baseRef); err != nil { + return nil, err + } + args := []string{"-c", "core.quotePath=false", "diff", "-U0", baseRef, "--"} args = append(args, paths...) out, err := runGitLimited(ctx, repoDir, args) if err != nil { @@ -98,6 +104,19 @@ func (g *ExecGitClient) DiffHunks(ctx context.Context, repoDir, baseRef string, return hunks, nil } +// validateBaseRef rejects base revisions that git would parse as command options. +// @intent block git argument injection through the caller-supplied base ref, which sits +// before the "--" separator and would otherwise be interpreted as a diff flag. +func validateBaseRef(baseRef string) error { + if baseRef == "" { + return fmt.Errorf("base ref must not be empty") + } + if strings.HasPrefix(baseRef, "-") { + return fmt.Errorf("invalid base ref %q: must not start with '-'", baseRef) + } + return nil +} + // runGitLimited runs a git subcommand with the default output size cap. // @intent share a single bounded git invocation helper across diff operations func runGitLimited(ctx context.Context, repoDir string, args []string) ([]byte, error) { diff --git a/internal/analysis/changes/git_test.go b/internal/analysis/changes/git_test.go index 8aabf72..cdf5870 100644 --- a/internal/analysis/changes/git_test.go +++ b/internal/analysis/changes/git_test.go @@ -29,6 +29,46 @@ func TestGitClient_ChangedFiles(t *testing.T) { } } +func TestGitClient_RejectsOptionLikeBaseRef(t *testing.T) { + dir := initTestRepo(t) + writeFile(t, dir, "hello.go", "package main\n") + gitCommit(t, dir, "initial") + + git := NewExecGitClient() + for _, base := range []string{"--output=/tmp/pwned", "-U9999", "--no-index"} { + if _, err := git.ChangedFiles(context.Background(), dir, base); err == nil { + t.Errorf("ChangedFiles accepted option-like base %q", base) + } + if _, err := git.DiffHunks(context.Background(), dir, base, []string{"hello.go"}); err == nil { + t.Errorf("DiffHunks accepted option-like base %q", base) + } + } +} + +func TestGitClient_ChangedFiles_NonASCIIPath(t *testing.T) { + dir := initTestRepo(t) + writeFile(t, dir, "한글.go", "package main\n") + gitCommit(t, dir, "initial") + writeFile(t, dir, "한글.go", "package main\n\nfunc Hello() {}\n") + + git := NewExecGitClient() + files, err := git.ChangedFiles(context.Background(), dir, "HEAD") + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || files[0] != "한글.go" { + t.Fatalf("expected unquoted 한글.go, got %v", files) + } + + hunks, err := git.DiffHunks(context.Background(), dir, "HEAD", []string{"한글.go"}) + if err != nil { + t.Fatal(err) + } + if len(hunks) == 0 || hunks[0].FilePath != "한글.go" { + t.Fatalf("expected hunk for unquoted 한글.go, got %+v", hunks) + } +} + func TestGitClient_DiffHunks(t *testing.T) { dir := initTestRepo(t) writeFile(t, dir, "hello.go", "package main\n\nfunc Old() {}\n") From b332f516b593592dc6735970f6564ffd1834c9f3 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:13 +0900 Subject: [PATCH 10/18] Require bearer auth on /status /status serializes sync-queue repo names, branches, and raw error strings; it was registered without the auth middleware that protects /mcp and /wiki/api/. /health and /ready stay open for probes. Co-Authored-By: Claude Fable 5 --- internal/server/serve.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/server/serve.go b/internal/server/serve.go index a68a2e7..d870731 100644 --- a/internal/server/serve.go +++ b/internal/server/serve.go @@ -112,9 +112,11 @@ func RunStreamableHTTP(rt *core.Runtime, srv *mcpgo.MCPServer, cfg Config, cache } return nil })) - mux.Handle("/status", StatusHandler(dbReadyCheck, cfg.WebhookAttemptTimeout, func() *webhook.SyncQueue { + // /status exposes repo names, branches, and raw error strings from the sync queue, + // so it requires the same bearer auth as /mcp; /health and /ready stay open for probes. + mux.Handle("/status", MCPAuthMiddleware(cfg.HTTPBearerToken, StatusHandler(dbReadyCheck, cfg.WebhookAttemptTimeout, func() *webhook.SyncQueue { return syncQueue - }, postprocessSummary)) + }, postprocessSummary))) if cfg.WikiDir != "" { wiki, err := wikiserver.New(wikiserver.Config{ From 2b2d3ae88faf5bd1fecb67898d2c3d458d44d233 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:13 +0900 Subject: [PATCH 11/18] Confine wiki doc API to the docs subtree, not the working directory resolveDocPath listed "." among its roots, so /wiki/api/doc and /wiki/api/context could read any file under the process CWD (.env, go.mod, source) as a "doc". Resolve shared docs against the docs/ directory itself for both relative and absolute paths. Also drop the orphaned retrieveResult type. Co-Authored-By: Claude Fable 5 --- internal/wikiserver/server.go | 25 ++++++++++++++++--------- internal/wikiserver/server_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/internal/wikiserver/server.go b/internal/wikiserver/server.go index 28cbb35..8849b09 100644 --- a/internal/wikiserver/server.go +++ b/internal/wikiserver/server.go @@ -552,13 +552,6 @@ type contextResponse struct { Items []contextItem `json:"items"` } -// @intent return retrieval metadata and optional bounded Markdown content to the Wiki UI. -type retrieveResult struct { - ragindex.RetrieveResult - Content string `json:"content,omitempty"` - ContentTruncated bool `json:"content_truncated,omitempty"` -} - // @intent return the resolved Wiki navigation target for one ccg:// ref. type refResponse struct { Namespace string `json:"namespace"` @@ -723,6 +716,8 @@ func readDocFile(resolved string) (string, string, error) { } // @intent resolve a generated doc path under approved docs, RAG, or namespace roots. +// @domainRule the working directory is only searched through its docs/ subtree; arbitrary +// repository files (config, source, secrets) must never be readable through the doc API. func (s *Server) resolveDocPath(namespace, docPath string) (string, error) { clean := filepath.Clean(docPath) if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { @@ -732,7 +727,9 @@ func (s *Server) resolveDocPath(namespace, docPath string) (string, error) { if namespace != ctxns.DefaultNamespace { roots = append(roots, filepath.Join(s.namespaceRoot, namespace)) } - roots = append(roots, ".", s.ragIndexDir, s.namespaceRoot) + // Generated shared docs live under ./docs (doc_path values carry the docs/ prefix), + // so the local root is the docs directory itself, not the bare working directory. + roots = append(roots, "docs", s.ragIndexDir, s.namespaceRoot) if filepath.IsAbs(clean) { for _, root := range roots { target, err := safeAbsolutePath(root, clean) @@ -746,7 +743,17 @@ func (s *Server) resolveDocPath(namespace, docPath string) (string, error) { return "", fs.ErrNotExist } for _, root := range roots { - target, err := safePath(root, clean) + rel := clean + if root == "docs" { + // doc_path values are docs-prefixed ("docs/pkg/file.md"); resolve the remainder + // inside the docs root so containment is enforced against docs/, not the CWD. + after, ok := strings.CutPrefix(clean, "docs"+string(os.PathSeparator)) + if !ok { + continue + } + rel = after + } + target, err := safePath(root, rel) if err != nil { continue } diff --git a/internal/wikiserver/server_test.go b/internal/wikiserver/server_test.go index 48fbf02..9a4b0d5 100644 --- a/internal/wikiserver/server_test.go +++ b/internal/wikiserver/server_test.go @@ -690,6 +690,32 @@ func TestAPI_DocReturnsNotFoundWithoutDBFallback(t *testing.T) { } } +func TestAPI_DocRejectsNonDocsFilesUnderCwd(t *testing.T) { + srv := newTestServer(t) + // A sensitive non-docs file in the working directory must not be readable as a "doc". + if err := os.WriteFile("secret.env", []byte("TOKEN=hunter2"), 0o644); err != nil { + t.Fatalf("write secret: %v", err) + } + for _, p := range []string{"secret.env", "./secret.env"} { + req := httptest.NewRequest(http.MethodGet, "/wiki/api/doc?namespace=default&path="+p, nil) + rec := httptest.NewRecorder() + srv.APIHandler().ServeHTTP(rec, req) + if rec.Code == http.StatusOK { + t.Fatalf("path %q: non-docs CWD file served as doc: %s", p, rec.Body.String()) + } + } + abs, err := filepath.Abs("secret.env") + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodGet, "/wiki/api/doc?namespace=default&path="+abs, nil) + rec := httptest.NewRecorder() + srv.APIHandler().ServeHTTP(rec, req) + if rec.Code == http.StatusOK { + t.Fatalf("absolute path: non-docs CWD file served as doc: %s", rec.Body.String()) + } +} + func TestAPI_DocRejectsTraversal(t *testing.T) { srv := newTestServer(t) req := httptest.NewRequest(http.MethodGet, "/wiki/api/doc?namespace=default&path=../secret.md", nil) From 3ab9d099c608ce556f2c703c580f0cfa3713dd06 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:29 +0900 Subject: [PATCH 12/18] Map the default namespace to the shared docs root in get_doc_content An explicit namespace:"default" resolved files under namespaces/default/ while resolvedRagIndexPath and retrieve_docs treat default as the shared docs root, so the same document was reachable with namespace omitted but not with it spelled out. Route default through the shared branch. Also drop the redundant retrieveDocsFromDB keep-alive var. Co-Authored-By: Claude Fable 5 --- internal/mcp/handler_docs.go | 6 +++--- internal/mcp/handler_docs_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/internal/mcp/handler_docs.go b/internal/mcp/handler_docs.go index d9d1c3e..1dad43e 100644 --- a/internal/mcp/handler_docs.go +++ b/internal/mcp/handler_docs.go @@ -216,8 +216,10 @@ func (h *handlers) getDocContent(ctx context.Context, request mcp.CallToolReques return mcp.NewToolResultError("invalid file_path: path traversal not allowed"), nil } + // The default namespace maps to the shared docs root, mirroring resolvedRagIndexPath + // and retrieve_docs; only named namespaces resolve under namespaces//. var resolvedPath string - if namespace != "" { + if namespace != "" && ctxns.Normalize(namespace) != ctxns.DefaultNamespace { if err := validateNamespacePath(namespace, filePath); err != nil { return mcp.NewToolResultError(err.Error()), nil } @@ -326,8 +328,6 @@ type retrieveDocsResponse = retrieval.Response // @intent keep MCP retrieve_docs result decoding compatible while sharing the canonical retrieval DTO. type retrieveDocsResult = retrieval.Result -var _ = (*handlers).retrieveDocsFromDB - // @intent build a retrieve_docs response from persisted graph nodes and annotation tags. // @requires SearchBackend and DB must be configured, and ctx must carry the desired namespace. // @ensures returned results are grouped one-per-file in first-seen FTS order and keep retrieve_docs' stable JSON shape. diff --git a/internal/mcp/handler_docs_test.go b/internal/mcp/handler_docs_test.go index ab0f3a8..0e863c0 100644 --- a/internal/mcp/handler_docs_test.go +++ b/internal/mcp/handler_docs_test.go @@ -295,6 +295,33 @@ func TestGetDocContent_PathTraversal(t *testing.T) { } } +func TestGetDocContent_DefaultNamespaceReadsSharedDocs(t *testing.T) { + deps := setupTestDeps(t) + deps.RagIndexDir = t.TempDir() + + content := "# Shared Doc\nfrom shared docs root" + docPath := filepath.Join(deps.RagIndexDir, "docs", "shared-doc.md") + if err := os.MkdirAll(filepath.Dir(docPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(docPath, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + // "default" must resolve to the shared docs root, matching resolvedRagIndexPath + // and retrieve_docs (contentNamespace maps default to shared), not namespaces/default/. + result := callTool(t, deps, "get_doc_content", map[string]any{ + "namespace": "default", + "file_path": "docs/shared-doc.md", + }) + if result.IsError { + t.Fatalf("default namespace should read shared docs, got error: %v", getTextContent(result)) + } + if got := getTextContent(result); got != content { + t.Errorf("want %q, got %q", content, got) + } +} + func TestGetDocContent_NotFound(t *testing.T) { deps := setupTestDeps(t) result := callTool(t, deps, "get_doc_content", map[string]any{ From a2782efad3d2de1c238105e55769bcbf077b8c7c Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:29 +0900 Subject: [PATCH 13/18] Skip dangling fallback edges instead of dropping the whole page find_suspect_fallback_edges returned an empty zero-value Result with a nil error whenever any edge endpoint node was missing, silently hiding every other suspect on the page. Skip edges with missing endpoints. Co-Authored-By: Claude Fable 5 --- internal/analysis/fallback/service.go | 8 +++-- internal/analysis/fallback/service_test.go | 38 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/internal/analysis/fallback/service.go b/internal/analysis/fallback/service.go index da23a3e..523f6c1 100644 --- a/internal/analysis/fallback/service.go +++ b/internal/analysis/fallback/service.go @@ -86,13 +86,17 @@ func (s *Service) FindSuspectsPage(ctx context.Context, opts Options) (Result, e results := make([]SuspectEdge, 0, len(edges)) for _, edge := range edges { source, err := s.store.GetNodeByID(ctx, edge.FromNodeID) - if err != nil || source == nil { + if err != nil { return Result{}, err } target, err := s.store.GetNodeByID(ctx, edge.ToNodeID) - if err != nil || target == nil { + if err != nil { return Result{}, err } + // A dangling edge (endpoint node missing) must not silently drop the whole page. + if source == nil || target == nil { + continue + } sourceAnn, err := s.store.GetAnnotation(ctx, source.ID) if err != nil { return Result{}, err diff --git a/internal/analysis/fallback/service_test.go b/internal/analysis/fallback/service_test.go index 94dbd42..f62d70a 100644 --- a/internal/analysis/fallback/service_test.go +++ b/internal/analysis/fallback/service_test.go @@ -63,6 +63,44 @@ func TestSuspectFallbackEdges_FlagsDisjointIntentAndDomainRule(t *testing.T) { } } +func TestSuspectFallbackEdges_SkipsEdgeWithMissingEndpointInsteadOfDroppingPage(t *testing.T) { + db := setupFallbackDB(t) + ctx := context.Background() + store := gormstore.New(db) + + source := model.Node{QualifiedName: "pkg.A", Kind: model.NodeKindFunction, Name: "A", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"} + target := model.Node{QualifiedName: "pkg.B", Kind: model.NodeKindFunction, Name: "B", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"} + if err := store.UpsertNodes(ctx, []model.Node{source, target}); err != nil { + t.Fatal(err) + } + sourceNode, _ := store.GetNode(ctx, "pkg.A") + targetNode, _ := store.GetNode(ctx, "pkg.B") + if err := store.UpsertEdges(ctx, []model.Edge{ + // dangling edge: endpoint node id does not exist + {FromNodeID: 99999, ToNodeID: targetNode.ID, Kind: model.EdgeKindFallbackCalls, Fingerprint: "a-dangling"}, + {FromNodeID: sourceNode.ID, ToNodeID: targetNode.ID, Kind: model.EdgeKindFallbackCalls, Fingerprint: "b-valid"}, + }); err != nil { + t.Fatal(err) + } + if err := store.UpsertAnnotation(ctx, &model.Annotation{NodeID: sourceNode.ID, Summary: "auth", Tags: []model.DocTag{{Kind: model.TagIntent, Value: "verify credentials", Ordinal: 0}}}); err != nil { + t.Fatal(err) + } + if err := store.UpsertAnnotation(ctx, &model.Annotation{NodeID: targetNode.ID, Summary: "billing", Tags: []model.DocTag{{Kind: model.TagIntent, Value: "render invoice", Ordinal: 0}}}); err != nil { + t.Fatal(err) + } + + results, err := New(db, store).FindSuspects(ctx, Options{}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("dangling edge must be skipped, not drop the whole page: got %d results", len(results)) + } + if results[0].Source.QualifiedName != "pkg.A" { + t.Fatalf("expected surviving valid edge, got %+v", results[0]) + } +} + func TestSuspectFallbackEdges_IgnoresOverlappingAnnotationContext(t *testing.T) { db := setupFallbackDB(t) ctx := context.Background() From 54ef7c5cb3c20244b8db5c9c1273ae77f2158717 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:29 +0900 Subject: [PATCH 14/18] Enforce namespace ownership on annotation create UpsertAnnotation's lookup was namespace-scoped but its create path was not, so a caller could attach an annotation to another namespace's node or to a nonexistent node id. Verify the node belongs to the caller's namespace inside the create transaction. Co-Authored-By: Claude Fable 5 --- internal/store/gormstore/gormstore.go | 12 ++++++++++ internal/store/gormstore/gormstore_test.go | 27 +++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/internal/store/gormstore/gormstore.go b/internal/store/gormstore/gormstore.go index c925508..a46d23f 100644 --- a/internal/store/gormstore/gormstore.go +++ b/internal/store/gormstore/gormstore.go @@ -4,6 +4,7 @@ package gormstore import ( "context" "errors" + "fmt" "path" "strings" @@ -516,6 +517,17 @@ func (s *Store) UpsertAnnotation(ctx context.Context, ann *model.Annotation) err if errors.Is(result.Error, gorm.ErrRecordNotFound) { return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // The read path is namespace-scoped; the create path must be too, or a caller + // could attach an annotation to another namespace's node (or a missing node). + var owned int64 + if err := tx.Model(&model.Node{}). + Where("id = ? AND namespace = ?", ann.NodeID, ns). + Count(&owned).Error; err != nil { + return trace.Wrap(err, "verify annotation node namespace") + } + if owned == 0 { + return fmt.Errorf("annotation node %d not found in namespace %q", ann.NodeID, ns) + } return tx.Create(ann).Error }) } diff --git a/internal/store/gormstore/gormstore_test.go b/internal/store/gormstore/gormstore_test.go index b172f71..30c151c 100644 --- a/internal/store/gormstore/gormstore_test.go +++ b/internal/store/gormstore/gormstore_test.go @@ -9,7 +9,7 @@ import ( "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" - gormschema "gorm.io/gorm/schema" + gormschema "gorm.io/gorm/schema" "github.com/tae2089/code-context-graph/internal/ctxns" "github.com/tae2089/code-context-graph/internal/model" @@ -626,6 +626,31 @@ func TestUpsertAnnotation_Insert(t *testing.T) { } } +func TestUpsertAnnotation_RejectsNodeOutsideNamespace(t *testing.T) { + s := setupTestDB(t) + ctx := context.Background() + + s.UpsertNodes(ctx, []model.Node{ + {QualifiedName: "pkg.F", Kind: model.NodeKindFunction, Name: "F", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}, + }) + node, _ := s.GetNode(ctx, "pkg.F") + + // Writing from another namespace must not create an annotation on a foreign node. + foreignCtx := ctxns.WithNamespace(context.Background(), "other-team") + err := s.UpsertAnnotation(foreignCtx, &model.Annotation{NodeID: node.ID, Summary: "cross-namespace write"}) + if err == nil { + t.Fatal("expected cross-namespace annotation create to be rejected") + } + if got, _ := s.GetAnnotation(ctx, node.ID); got != nil { + t.Fatalf("annotation must not exist after rejected write, got %+v", got) + } + + // Nonexistent node id must also be rejected instead of creating an orphan row. + if err := s.UpsertAnnotation(ctx, &model.Annotation{NodeID: 99999, Summary: "orphan"}); err == nil { + t.Fatal("expected annotation create for missing node to be rejected") + } +} + func TestUpsertAnnotation_Update(t *testing.T) { s := setupTestDB(t) ctx := context.Background() From 8f1ef58e5f52620fb5cf2a0bc4fcc60fddf747ea Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:23:45 +0900 Subject: [PATCH 15/18] Repair broken and flaky tests exposed by live-Postgres and load runs - RebuildNodes tests staged a stale tsv via UPDATE, but the BEFORE UPDATE trigger recomputed tsv from content and erased the stale state, failing both tests against a real Postgres. Disable the trigger during staging so RebuildNodes' own SQL is what gets verified. - The rapid-push dedup test raced three Adds against an already-running worker and flaked under parallel load. Hold the single worker with a gated blocker repo so all pushes deterministically arrive while queued. - main_postgres_test referenced migrateSchemaVersion, a type renamed to migration.MigrationSchemaVersion; the postgres-tagged file no longer compiled (unnoticed because CI never builds that tag). Co-Authored-By: Claude Fable 5 --- cmd/ccg/main_postgres_test.go | 4 ++-- internal/store/search/postgres_test.go | 16 ++++++++++++++ internal/webhook/syncqueue_test.go | 29 +++++++++++++++++--------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/cmd/ccg/main_postgres_test.go b/cmd/ccg/main_postgres_test.go index b1883f1..4d088b5 100644 --- a/cmd/ccg/main_postgres_test.go +++ b/cmd/ccg/main_postgres_test.go @@ -157,7 +157,7 @@ func TestRunMigrations_PostgresDownRestoresNullableColumns(t *testing.T) { t.Fatalf("run down migration: %v", err) } - var version migrateSchemaVersion + var version migration.MigrationSchemaVersion if err := db.Table("schema_migrations").First(&version).Error; err != nil { t.Fatalf("load schema version: %v", err) } @@ -190,7 +190,7 @@ func TestRunMigrations_PostgresDownFromVersionThreeDropsPolicyTables(t *testing. t.Fatalf("run down migration: %v", err) } - var version migrateSchemaVersion + var version migration.MigrationSchemaVersion if err := db.Table("schema_migrations").First(&version).Error; err != nil { t.Fatalf("load schema version: %v", err) } diff --git a/internal/store/search/postgres_test.go b/internal/store/search/postgres_test.go index faca472..e66ac4b 100644 --- a/internal/store/search/postgres_test.go +++ b/internal/store/search/postgres_test.go @@ -133,6 +133,15 @@ func TestPostgresFTS_Rebuild(t *testing.T) { } } +// disableTSVTrigger turns off the tsv BEFORE INSERT/UPDATE trigger so tests can stage +// stale tsv values by hand. Tables are dropped per test, so re-enabling is unnecessary. +func disableTSVTrigger(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec("ALTER TABLE search_documents DISABLE TRIGGER trg_search_documents_tsv").Error; err != nil { + t.Fatalf("disable tsv trigger: %v", err) + } +} + func TestPostgresFTS_RebuildNodes_RefreshesOnlyScopedRows(t *testing.T) { db := setupPostgresDB(t) backend := NewPostgresBackend() @@ -157,6 +166,10 @@ func TestPostgresFTS_RebuildNodes_RefreshesOnlyScopedRows(t *testing.T) { if err := db.Create(&model.SearchDocument{Namespace: "ns-b", NodeID: foreign.ID, Content: "fresh foreign", Language: "go"}).Error; err != nil { t.Fatal(err) } + // The tsv BEFORE UPDATE trigger would recompute tsv from content and overwrite the + // manual stale marker; disable it so the stale state actually exists and RebuildNodes' + // own UPDATE statement is what gets verified. + disableTSVTrigger(t, db) if err := db.Exec("UPDATE search_documents SET tsv = to_tsvector('simple', 'stale')").Error; err != nil { t.Fatal(err) } @@ -193,6 +206,9 @@ func TestPostgresFTS_RebuildNodes_EmptyScopeIsNoOp(t *testing.T) { if err := db.Create(&model.SearchDocument{Namespace: ctxns.DefaultNamespace, NodeID: node.ID, Content: "fresh keep", Language: "go"}).Error; err != nil { t.Fatal(err) } + // See TestPostgresFTS_RebuildNodes_RefreshesOnlyScopedRows: the trigger must be off + // for the stale marker to persist. + disableTSVTrigger(t, db) if err := db.Exec("UPDATE search_documents SET tsv = to_tsvector('simple', 'stale')").Error; err != nil { t.Fatal(err) } diff --git a/internal/webhook/syncqueue_test.go b/internal/webhook/syncqueue_test.go index 89018c3..7ee818c 100644 --- a/internal/webhook/syncqueue_test.go +++ b/internal/webhook/syncqueue_test.go @@ -14,35 +14,44 @@ import ( ) func TestSyncQueue_DeduplicatesRapidPushes(t *testing.T) { - var callCount atomic.Int32 - done := make(chan struct{}) + var svcCalls atomic.Int32 + gate := make(chan struct{}) + svcDone := make(chan struct{}) handler := func(_ context.Context, repoFullName, cloneURL, branch string) error { - callCount.Add(1) - time.Sleep(50 * time.Millisecond) - if callCount.Load() == 1 { - close(done) + switch repoFullName { + case "org/blocker": + <-gate + case "org/svc": + if svcCalls.Add(1) == 1 { + close(svcDone) + } } return nil } - q := NewSyncQueue(2, handler) + // A single worker held by the gated blocker guarantees all three pushes arrive while + // org/svc is queued (never processing), so dedup must collapse them into one run. + // Racing the pushes against a free worker made this test flaky under parallel load. + q := NewSyncQueue(1, handler) defer q.Shutdown() + q.Add(context.Background(), "org/blocker", "https://github.com/org/blocker.git", "main") q.Add(context.Background(), "org/svc", "https://github.com/org/svc.git", "main") q.Add(context.Background(), "org/svc", "https://github.com/org/svc.git", "main") q.Add(context.Background(), "org/svc", "https://github.com/org/svc.git", "main") + close(gate) select { - case <-done: + case <-svcDone: case <-time.After(5 * time.Second): t.Fatal("timed out waiting for handler") } + // Grace window so an erroneous duplicate run would surface before the assertion. time.Sleep(100 * time.Millisecond) - got := callCount.Load() - if got != 1 { + if got := svcCalls.Load(); got != 1 { t.Errorf("handler called %d times, want 1 (dedup failed)", got) } } From b05b3aff4e82336cee8efb2c2a5903dcb57bb51b Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:24:05 +0900 Subject: [PATCH 16/18] Remove dead code and fix gofmt drift Delete verified-unused symbols: existingFilesMissingFrom, importPackageNames, rebuildSearch, newParsedBuildEdgeBatch (service); countNonIgnored, lintRuleDoc (cli/lint). Run gofmt -w over the 13 drifted files. Co-Authored-By: Claude Fable 5 --- internal/analysis/flows/flows.go | 2 +- internal/annotation/normalizer.go | 2 +- internal/cli/lint.go | 13 -- internal/parse/binder_test.go | 116 +++++++++--------- .../binding_gap_cross_language_test.go | 12 +- .../binding_gap_go_directives_test.go | 4 +- .../binding_gap_integration_test.go | 30 ++--- .../parse/treesitter/binding_gap_p1_test.go | 24 ++-- .../parse/treesitter/p1_ast_probe_test.go | 18 +-- .../parse/treesitter/package_discovery.go | 8 +- .../treesitter/package_discovery_test.go | 3 +- .../python_docstring_variants_test.go | 84 ++++++------- internal/pathutil/exclude_test.go | 2 +- internal/service/indexer.go | 41 ------- 14 files changed, 154 insertions(+), 205 deletions(-) diff --git a/internal/analysis/flows/flows.go b/internal/analysis/flows/flows.go index ffa50d4..1e569a7 100644 --- a/internal/analysis/flows/flows.go +++ b/internal/analysis/flows/flows.go @@ -143,7 +143,7 @@ func (t *Tracer) TraceFlowBounded(ctx context.Context, startNodeID uint, opts Tr Truncated: truncated, MaxNodes: opts.MaxNodes, ReturnedNodes: len(members), - ContainsFallbackCalls: containsFallbackCalls, + ContainsFallbackCalls: containsFallbackCalls, FallbackEdgesCount: fallbackEdges, }, nil } diff --git a/internal/annotation/normalizer.go b/internal/annotation/normalizer.go index ac0a8a0..7dce1f4 100644 --- a/internal/annotation/normalizer.go +++ b/internal/annotation/normalizer.go @@ -84,7 +84,7 @@ func stripBlockDelimiters(text string, language string) string { } // stripPythonDocstringDelimiters removes triple-quote wrappers from a Python docstring body. -// @intent expose the raw docstring text by trying both """ and ''' triple-quote forms. +// @intent expose the raw docstring text by trying both """ and ”' triple-quote forms. func stripPythonDocstringDelimiters(text string) (string, bool) { for _, quote := range []string{"\"\"\"", "'''"} { if stripped, ok := stripPythonQuotedString(text, quote); ok { diff --git a/internal/cli/lint.go b/internal/cli/lint.go index 1fbd75a..a0c1321 100644 --- a/internal/cli/lint.go +++ b/internal/cli/lint.go @@ -17,13 +17,6 @@ import ( "github.com/tae2089/code-context-graph/internal/pathutil" ) -// countNonIgnored counts lint issues not covered by an ignore rule in .ccg.yaml. -// @intent strict 모드에서 실제 실패로 간주할 lint 항목만 다시 집계한다. -// @domainRule action: ignore로 선언된 규칙은 strict 실패 수에서 제외한다. -func countNonIgnored(report *docs.LintReport) int { - return countNonIgnoredWithRules(report, configuredLintRules()) -} - // normalizeLintCategory maps legacy or variant category names to canonical forms. // @intent ensure rule matching uses consistent category keys regardless of input spelling // @domainRule "deadref" and "drifted" are accepted aliases for "dead-ref" and "drift" @@ -261,12 +254,6 @@ func lintRuleFromMapLookup(lookup func(string) (any, bool)) (lintRule, bool) { return rule, rule.Pattern != "" || rule.Category != "" || rule.Action != "" } -// lintRuleDoc is the top-level YAML structure for a standalone rules file. -// @intent represent the on-disk format used when rules are stored outside .ccg.yaml -type lintRuleDoc struct { - Rules []lintRule `yaml:"rules"` -} - // lintRule is a single suppression or warn rule entry in .ccg.yaml or auto-rules.yaml. // @intent carry the pattern, category, and action that determine how a lint finding is handled // @domainRule auto:true marks rules generated by the Twice Rule engine, not written by the user diff --git a/internal/parse/binder_test.go b/internal/parse/binder_test.go index 9261d76..a0abbcf 100644 --- a/internal/parse/binder_test.go +++ b/internal/parse/binder_test.go @@ -111,10 +111,11 @@ func TestBinder_MultipleDeclarations(t *testing.T) { // 바인딩이 성공함을 검증합니다. // // fixture 기준: -// Line 1-4: docstring ("""...""") -// Line 5: @app.route('/api/user') -// Line 6: @login_required -// Line 7: def get_user(): ← tree-sitter StartLine 예상 +// +// Line 1-4: docstring ("""...""") +// Line 5: @app.route('/api/user') +// Line 6: @login_required +// Line 7: def get_user(): ← tree-sitter StartLine 예상 // // gap = 7 - 4 = 3 ≤ maxGap(3) → 바인딩 성공 func TestBinder_PythonDecoratorGap_BindsWithinMaxGap(t *testing.T) { @@ -188,11 +189,12 @@ func TestBinder_JavaAnnotationGap_PassthroughBinds(t *testing.T) { // 바인딩이 성공함을 검증합니다. // // fixture (attribute_gap.rs) 기준: -// Line 1: /// @intent 비동기 main 진입점 -// Line 2: /// @sideEffect 런타임 초기화 -// Line 3: #[tokio::main] -// Line 4: #[allow(dead_code)] -// Line 5: async fn main() { ← tree-sitter StartLine +// +// Line 1: /// @intent 비동기 main 진입점 +// Line 2: /// @sideEffect 런타임 초기화 +// Line 3: #[tokio::main] +// Line 4: #[allow(dead_code)] +// Line 5: async fn main() { ← tree-sitter StartLine // // gap = 5 - 2 = 3 ≤ maxGap(3) → 바인딩 성공 func TestBinder_RustAttributeGap_BindsWithinMaxGap(t *testing.T) { @@ -226,10 +228,11 @@ func TestBinder_RustAttributeGap_BindsWithinMaxGap(t *testing.T) { // 바인딩이 성공함을 검증합니다. // // fixture (attribute_gap.c) 기준: -// Line 1-3: /** ... */ Doxygen -// Line 4: __attribute__((always_inline)) -// Line 5: __attribute__((nonnull)) -// Line 6: static inline int add(int a, int b) { ← tree-sitter StartLine 예상 +// +// Line 1-3: /** ... */ Doxygen +// Line 4: __attribute__((always_inline)) +// Line 5: __attribute__((nonnull)) +// Line 6: static inline int add(int a, int b) { ← tree-sitter StartLine 예상 // // gap = 6 - 3 = 3 ≤ maxGap(3) → 바인딩 성공 func TestBinder_CAttributeGap_BindsWithinMaxGap(t *testing.T) { @@ -292,7 +295,6 @@ func TestBinder_GapWithCodeBetween_NoBinding(t *testing.T) { } } - // ============================================================================= // Look-Between 동적 바인딩 테스트 // ============================================================================= @@ -320,10 +322,10 @@ func TestBinder_LookBetween_BlankLinesOnly_Binds(t *testing.T) { } sourceLines := []string{ "// @intent 빈 줄 사이 바인딩", // line 1 - "", // line 2 - "", // line 3 - "", // line 4 - "func MyFunc() {", // line 5 + "", // line 2 + "", // line 3 + "", // line 4 + "func MyFunc() {", // line 5 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -357,8 +359,8 @@ func TestBinder_LookBetween_CodeBetween_NoBinding(t *testing.T) { } sourceLines := []string{ "// @intent 이 주석은 바인딩 안 됨", // line 1 - "var x = 42", // line 2 - 코드! - "func MyFunc() {", // line 3 + "var x = 42", // line 2 - 코드! + "func MyFunc() {", // line 3 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -395,8 +397,8 @@ func TestBinder_LookBetween_Adjacent_Gap1_Binds(t *testing.T) { } sourceLines := []string{ "// @intent 인접 바인딩", // line 1 - "// (continued)", // line 2 - "func MyFunc() {", // line 3 + "// (continued)", // line 2 + "func MyFunc() {", // line 3 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -439,9 +441,9 @@ func TestBinder_LookBetween_WhitespaceOnlyLines_Binds(t *testing.T) { } sourceLines := []string{ "// @intent 공백문자만 있는 줄", // line 1 - " ", // line 2 - spaces only - "\t\t", // line 3 - tabs only - "func MyFunc() {", // line 4 + " ", // line 2 - spaces only + "\t\t", // line 3 - tabs only + "func MyFunc() {", // line 4 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -477,9 +479,9 @@ func TestBinder_Passthrough_PythonDecorators_Binds(t *testing.T) { } sourceLines := []string{ "# @intent 사용자 조회 엔드포인트", // line 1 - "@app.route('/api/user')", // line 2 - decorator - "@login_required", // line 3 - decorator - "def get_user():", // line 4 + "@app.route('/api/user')", // line 2 - decorator + "@login_required", // line 3 - decorator + "def get_user():", // line 4 } bindings := b.Bind(comments, nodes, "python", sourceLines) @@ -511,13 +513,13 @@ func TestBinder_Passthrough_JavaAnnotations_Binds(t *testing.T) { {Name: "UserService", Kind: model.NodeKindClass, StartLine: 7, EndLine: 20}, } sourceLines := []string{ - "/**", // line 1 - " * @intent 사용자 서비스", // line 2 - " */", // line 3 - "@Service", // line 4 - annotation - "@Transactional", // line 5 - annotation - "@RequiredArgsConstructor", // line 6 - annotation - "public class UserService {", // line 7 + "/**", // line 1 + " * @intent 사용자 서비스", // line 2 + " */", // line 3 + "@Service", // line 4 - annotation + "@Transactional", // line 5 - annotation + "@RequiredArgsConstructor", // line 6 - annotation + "public class UserService {", // line 7 } bindings := b.Bind(comments, nodes, "java", sourceLines) @@ -549,11 +551,11 @@ func TestBinder_Passthrough_RustAttributes_Binds(t *testing.T) { {Name: "main", Kind: model.NodeKindFunction, StartLine: 5, EndLine: 10}, } sourceLines := []string{ - "/// @intent 비동기 main 진입점", // line 1 - "/// @sideEffect 런타임 초기화", // line 2 - "#[tokio::main]", // line 3 - Rust attribute - "#[allow(dead_code)]", // line 4 - Rust attribute - "async fn main() {", // line 5 + "/// @intent 비동기 main 진입점", // line 1 + "/// @sideEffect 런타임 초기화", // line 2 + "#[tokio::main]", // line 3 - Rust attribute + "#[allow(dead_code)]", // line 4 - Rust attribute + "async fn main() {", // line 5 } bindings := b.Bind(comments, nodes, "rust", sourceLines) @@ -584,12 +586,12 @@ func TestBinder_Passthrough_CAttributes_Binds(t *testing.T) { {Name: "add", Kind: model.NodeKindFunction, StartLine: 6, EndLine: 10}, } sourceLines := []string{ - "/**", // line 1 - " * @intent 항상 인라인되는 덧셈", // line 2 - " */", // line 3 - "__attribute__((always_inline))", // line 4 - C attribute - "[[nodiscard]]", // line 5 - C++17 attribute - "static inline int add(int a, int b) {", // line 6 + "/**", // line 1 + " * @intent 항상 인라인되는 덧셈", // line 2 + " */", // line 3 + "__attribute__((always_inline))", // line 4 - C attribute + "[[nodiscard]]", // line 5 - C++17 attribute + "static inline int add(int a, int b) {", // line 6 } bindings := b.Bind(comments, nodes, "c", sourceLines) @@ -620,10 +622,10 @@ func TestBinder_Passthrough_OtherComments_Binds(t *testing.T) { {Name: "MyFunc", Kind: model.NodeKindFunction, StartLine: 4, EndLine: 10}, } sourceLines := []string{ - "// @intent 주석 사이 바인딩", // line 1 - "// TODO: 나중에 리팩토링", // line 2 - comment + "// @intent 주석 사이 바인딩", // line 1 + "// TODO: 나중에 리팩토링", // line 2 - comment "// NOTE: 이 함수는 deprecated 예정", // line 3 - comment - "func MyFunc() {", // line 4 + "func MyFunc() {", // line 4 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -657,11 +659,11 @@ func TestBinder_Passthrough_Mixed_Binds(t *testing.T) { } sourceLines := []string{ "# @intent 혼합 passthrough 테스트", // line 1 - "# type: ignore", // line 2 - comment - "", // line 3 - blank - "@app.route('/test')", // line 4 - decorator - "@requires_auth", // line 5 - decorator - "def handler():", // line 6 + "# type: ignore", // line 2 - comment + "", // line 3 - blank + "@app.route('/test')", // line 4 - decorator + "@requires_auth", // line 5 - decorator + "def handler():", // line 6 } bindings := b.Bind(comments, nodes, "python", sourceLines) @@ -693,9 +695,9 @@ func TestBinder_Passthrough_DecoratorPlusCode_NoBinding(t *testing.T) { } sourceLines := []string{ "# @intent 이건 바인딩 안 됨", // line 1 - "@decorator", // line 2 - decorator (passthrough) - "x = 42", // line 3 - actual code! - "def handler():", // line 4 + "@decorator", // line 2 - decorator (passthrough) + "x = 42", // line 3 - actual code! + "def handler():", // line 4 } bindings := b.Bind(comments, nodes, "python", sourceLines) diff --git a/internal/parse/treesitter/binding_gap_cross_language_test.go b/internal/parse/treesitter/binding_gap_cross_language_test.go index 878fa87..69dfd46 100644 --- a/internal/parse/treesitter/binding_gap_cross_language_test.go +++ b/internal/parse/treesitter/binding_gap_cross_language_test.go @@ -9,8 +9,8 @@ // 테스트 정책: // - expectBound=true → Green 계약. 정상 바인딩되어야 하며, 실패 시 회귀(normalizer/walker/binder). // - expectBound=false → Red 계약. 현재 tree-sitter grammar 특성으로 바인딩이 누락됨을 -// "명시적으로" 고정. 구현이 개선되어 우연히 바인딩되면 테스트가 실패하므로 -// 그때 Green으로 승격하고 skipReason과 walker 보정을 함께 정리한다. +// "명시적으로" 고정. 구현이 개선되어 우연히 바인딩되면 테스트가 실패하므로 +// 그때 Green으로 승격하고 skipReason과 walker 보정을 함께 정리한다. // // Helper 함수: // - binderFromWalkerComments, logNodeInfo, logCommentInfo 는 @@ -338,10 +338,10 @@ data class User(val name: String) } // Phase 2: 바인딩 시도. - binder := parse.NewBinder() - bindings := binder.Bind( - binderFromWalkerComments(walkerComments), nodes, tc.spec.Name, strings.Split(tc.source, "\n"), - ) + binder := parse.NewBinder() + bindings := binder.Bind( + binderFromWalkerComments(walkerComments), nodes, tc.spec.Name, strings.Split(tc.source, "\n"), + ) var target *parse.Binding for i := range bindings { diff --git a/internal/parse/treesitter/binding_gap_go_directives_test.go b/internal/parse/treesitter/binding_gap_go_directives_test.go index 42da704..e39d1b9 100644 --- a/internal/parse/treesitter/binding_gap_go_directives_test.go +++ b/internal/parse/treesitter/binding_gap_go_directives_test.go @@ -68,8 +68,8 @@ func TestWalkerBinder_Go_DirectivePollution(t *testing.T) { t.Log("--- 파싱된 CommentBlock ---") logCommentInfo(t, walkerComments) - b := parse.NewBinder() - bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, "go", strings.Split(string(content), "\n")) + b := parse.NewBinder() + bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, "go", strings.Split(string(content), "\n")) var intentValue string var bound bool diff --git a/internal/parse/treesitter/binding_gap_integration_test.go b/internal/parse/treesitter/binding_gap_integration_test.go index f693c3c..c14cf38 100644 --- a/internal/parse/treesitter/binding_gap_integration_test.go +++ b/internal/parse/treesitter/binding_gap_integration_test.go @@ -4,9 +4,9 @@ // 여러 줄 붙어 있으면 @intent docstring/주석이 심볼 노드에 바인딩되지 않는다. // // 이 파일의 테스트는 각 언어별로: -// 1. 실제 tree-sitter로 fixture를 파싱 -// 2. 심볼 노드의 실제 StartLine을 t.Logf로 출력 -// 3. Binder.Bind 결과를 검증하여 현재 실패 동작을 명시적으로 기록 +// 1. 실제 tree-sitter로 fixture를 파싱 +// 2. 심볼 노드의 실제 StartLine을 t.Logf로 출력 +// 3. Binder.Bind 결과를 검증하여 현재 실패 동작을 명시적으로 기록 // // 테스트 이름에 _CurrentlyFailsBinding 접미사를 붙여 현재 상태를 명시합니다. package treesitter @@ -421,15 +421,17 @@ func TestWalkerBinder_CAttribute_CurrentlyFailsBinding(t *testing.T) { // # 라인 주석 + 데코레이터 2개 시나리오로 바인딩 실패를 검증합니다. // // fixture (decorator_gap_comment.py) 기준: -// Line 1: # @intent 사용자 조회 엔드포인트 -// Line 2: # @domainRule 인증 필요 -// Line 3: @app.route('/api/user') -// Line 4: @login_required -// Line 5: def get_user(): ← tree-sitter StartLine +// +// Line 1: # @intent 사용자 조회 엔드포인트 +// Line 2: # @domainRule 인증 필요 +// Line 3: @app.route('/api/user') +// Line 4: @login_required +// Line 5: def get_user(): ← tree-sitter StartLine // // 주석EndLine=2, get_user StartLine=5, gap=3 > maxGap(2) → 바인딩 실패 예상 // (단, Python이 decorated_definition으로 StartLine을 데코레이터 첫 줄=3으로 잡으면 -// gap=3-2=1이 되어 바인딩 성공 가능성도 있음) +// +// gap=3-2=1이 되어 바인딩 성공 가능성도 있음) func TestWalkerBinder_PythonDecoratorHashComment_CurrentlyFailsBinding(t *testing.T) { content := readFixture(t, "python", "decorator_gap_comment.py") t.Logf("fixture 내용:\n%s", string(content)) @@ -517,11 +519,11 @@ func TestWalkerBinder_PythonDecoratorHashComment_CurrentlyFailsBinding(t *testin // 이 테스트는 항상 통과하며, 진단 데이터만 수집합니다. func TestWalkerBinder_GapDiagnosis_AllLanguages(t *testing.T) { type langCase struct { - spec *LangSpec - lang string - filename string - symName string - symKind model.NodeKind + spec *LangSpec + lang string + filename string + symName string + symKind model.NodeKind fixtureExt string } diff --git a/internal/parse/treesitter/binding_gap_p1_test.go b/internal/parse/treesitter/binding_gap_p1_test.go index ae3feb9..4d5f196 100644 --- a/internal/parse/treesitter/binding_gap_p1_test.go +++ b/internal/parse/treesitter/binding_gap_p1_test.go @@ -9,10 +9,10 @@ // - 언어별 tree-sitter 문법이 메타 표식을 심볼 노드에 포함하는지 확인 // // 검증 대상 4개 분류: -// 1. 심볼 StartLine이 메타 표식 줄을 포함하는가 (Java/C처럼 wrapper가 흡수) -// 2. @intent 태그가 정상 파싱되는가 (normalizer 이슈 없는가) -// 3. gap이 maxGap(=2) 이내인가 -// 4. 최종적으로 Binder가 @intent를 바인딩하는가 +// 1. 심볼 StartLine이 메타 표식 줄을 포함하는가 (Java/C처럼 wrapper가 흡수) +// 2. @intent 태그가 정상 파싱되는가 (normalizer 이슈 없는가) +// 3. gap이 maxGap(=2) 이내인가 +// 4. 최종적으로 Binder가 @intent를 바인딩하는가 package treesitter import ( @@ -215,11 +215,11 @@ func TestWalkerBinder_P1_AllLanguages_Summary(t *testing.T) { } type row struct { - label string - startLine int - intentBound bool - minGap int - found bool + label string + startLine int + intentBound bool + minGap int + found bool } var rows []row @@ -252,9 +252,9 @@ func TestWalkerBinder_P1_AllLanguages_Summary(t *testing.T) { } } - // 바인딩 체크 - b := parse.NewBinder() - bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, tc.lang, strings.Split(string(content), "\n")) + // 바인딩 체크 + b := parse.NewBinder() + bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, tc.lang, strings.Split(string(content), "\n")) for _, binding := range bindings { if binding.Node.Name != tc.symName || binding.Node.Kind != tc.symKind { continue diff --git a/internal/parse/treesitter/p1_ast_probe_test.go b/internal/parse/treesitter/p1_ast_probe_test.go index 4ba4e87..a95cc6e 100644 --- a/internal/parse/treesitter/p1_ast_probe_test.go +++ b/internal/parse/treesitter/p1_ast_probe_test.go @@ -69,15 +69,15 @@ func TestP1_AST_NodeTypes(t *testing.T) { } for _, tc := range cases { - t.Run(tc.label, func(t *testing.T) { - content := readFixture(t, tc.lang, tc.filename) - w := NewWalker(tc.spec) - parser := w.acquireParser() - defer w.releaseParser(parser) - tree, err := parser.ParseCtx(context.Background(), nil, content) - if err != nil { - t.Fatalf("[%s] 파싱 실패: %v", tc.label, err) - } + t.Run(tc.label, func(t *testing.T) { + content := readFixture(t, tc.lang, tc.filename) + w := NewWalker(tc.spec) + parser := w.acquireParser() + defer w.releaseParser(parser) + tree, err := parser.ParseCtx(context.Background(), nil, content) + if err != nil { + t.Fatalf("[%s] 파싱 실패: %v", tc.label, err) + } defer tree.Close() root := tree.RootNode() diff --git a/internal/parse/treesitter/package_discovery.go b/internal/parse/treesitter/package_discovery.go index 8ab8f94..7120e69 100644 --- a/internal/parse/treesitter/package_discovery.go +++ b/internal/parse/treesitter/package_discovery.go @@ -81,7 +81,7 @@ func (PythonPackageDiscovery) DiscoverPackages(ctx context.Context, opts Package // @intent create package nodes for package.json paths and tsconfig alias paths that imports can target. func (TypeScriptPackageDiscovery) DiscoverPackages(ctx context.Context, opts PackageDiscoveryOptions) (map[string]PackageInfo, error) { return discoverNodePackages(ctx, opts, nodePackageDiscoveryConfig{ - language: "typescript", + language: "typescript", extensions: []string{".ts", ".tsx"}, packageJSON: true, tsconfigAlias: true, @@ -92,7 +92,7 @@ func (TypeScriptPackageDiscovery) DiscoverPackages(ctx context.Context, opts Pac // @intent create package nodes for JavaScript directories using package.json-derived import paths. func (JavaScriptPackageDiscovery) DiscoverPackages(ctx context.Context, opts PackageDiscoveryOptions) (map[string]PackageInfo, error) { return discoverNodePackages(ctx, opts, nodePackageDiscoveryConfig{ - language: "javascript", + language: "javascript", extensions: []string{".js", ".jsx", ".mjs", ".cjs"}, packageJSON: true, }) @@ -349,7 +349,7 @@ func pathBaseName(value, sep string) string { // nodePackageDiscoveryConfig describes one Node.js-family package discovery mode. // @intent share package.json and tsconfig-based directory discovery across TypeScript and JavaScript. type nodePackageDiscoveryConfig struct { - language string + language string extensions []string packageJSON bool tsconfigAlias bool @@ -358,7 +358,7 @@ type nodePackageDiscoveryConfig struct { // nodePackageJSON is the subset of package.json used for import-path discovery. // @intent keep package metadata parsing minimal while deriving package-node qualified names. type nodePackageJSON struct { - Name string `json:"name"` + Name string `json:"name"` Workspaces json.RawMessage `json:"workspaces"` } diff --git a/internal/parse/treesitter/package_discovery_test.go b/internal/parse/treesitter/package_discovery_test.go index 208686a..63d529c 100644 --- a/internal/parse/treesitter/package_discovery_test.go +++ b/internal/parse/treesitter/package_discovery_test.go @@ -756,8 +756,7 @@ func TestPackageDiscoveryOrDefault_DefaultsForUnimplementedLanguages(t *testing. tests := []struct { name string spec *LangSpec - }{ - } + }{} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/parse/treesitter/python_docstring_variants_test.go b/internal/parse/treesitter/python_docstring_variants_test.go index 1b0e656..809bbb5 100644 --- a/internal/parse/treesitter/python_docstring_variants_test.go +++ b/internal/parse/treesitter/python_docstring_variants_test.go @@ -1,6 +1,6 @@ // Python docstring 2차 실측 테스트. // -// 목적: tree-sitter-python이 다양한 docstring 변형(함수/클래스/모듈, """/''', 한 줄, prefix)을 +// 목적: tree-sitter-python이 다양한 docstring 변형(함수/클래스/모듈, """/”', 한 줄, prefix)을 // 어떤 AST 노드 구조로 파싱하는지 정확히 측정하고, // 현재 collectComments 로직이 이들을 CommentBlock으로 수집하는지 검증한다. // @@ -145,13 +145,13 @@ func parseRawTree(t *testing.T, content []byte) *sitter.Node { // stringNodeDetail은 string 노드의 상세 정보를 분석한다. type stringNodeDetail struct { - nodeType string // "string", "string_content", 기타 - startLine int - endLine int - isOneLiner bool - prefixToken string // "r", "f", "b", "rb" 등 (없으면 "") - quoteStyle string // `"""` or `'''` or `"` or `'` - rawContent string // 노드 Content의 첫 80자 + nodeType string // "string", "string_content", 기타 + startLine int + endLine int + isOneLiner bool + prefixToken string // "r", "f", "b", "rb" 등 (없으면 "") + quoteStyle string // `"""` or `'''` or `"` or `'` + rawContent string // 노드 Content의 첫 80자 } func analyzeStringNode(node *sitter.Node, content []byte) stringNodeDetail { @@ -224,17 +224,17 @@ func analyzeStringNode(node *sitter.Node, content []byte) stringNodeDetail { // ───────────────────────────────────────────────────────────── type docstringMeasurement struct { - fixture string - nodeType string // string 노드 타입 - parentChain []string // 부모 체인 (루트 방향) - startLine int - endLine int - prefixToken string - quoteStyle string - isOneLiner bool - inCommentBlock bool // 현재 CommentBlock에 포함되는가 - intentBound bool // @intent 바인딩 성공 여부 - bindingNote string // 추가 메모 + fixture string + nodeType string // string 노드 타입 + parentChain []string // 부모 체인 (루트 방향) + startLine int + endLine int + prefixToken string + quoteStyle string + isOneLiner bool + inCommentBlock bool // 현재 CommentBlock에 포함되는가 + intentBound bool // @intent 바인딩 성공 여부 + bindingNote string // 추가 메모 } func (m docstringMeasurement) parentChainStr() string { @@ -253,7 +253,7 @@ func (m docstringMeasurement) parentChainStr() string { func measureDocstringFixture( t *testing.T, filename string, - targetSymName string, // 바인딩 대상 심볼 이름 ("" 이면 file 노드) + targetSymName string, // 바인딩 대상 심볼 이름 ("" 이면 file 노드) targetSymKind model.NodeKind, ) docstringMeasurement { t.Helper() @@ -408,10 +408,10 @@ func TestPythonDocstring_FuncDouble(t *testing.T) { } } -// TestPythonDocstring_FuncSingle 은 함수 본문 내 '''...''' docstring을 실측한다. +// TestPythonDocstring_FuncSingle 은 함수 본문 내 ”'...”' docstring을 실측한다. // // 검증 포인트: -// - """와 '''가 동일한 tree-sitter 노드 타입("string")으로 처리되는가 +// - """와 ”'가 동일한 tree-sitter 노드 타입("string")으로 처리되는가 func TestPythonDocstring_FuncSingle(t *testing.T) { m := measureDocstringFixture(t, "docstring_func_single.py", "get_user", model.NodeKindFunction) @@ -596,10 +596,10 @@ func TestPythonDocstring_Prefix(t *testing.T) { // 이 테스트 자체는 항상 Pass (진단 전용). func TestPythonDocstring_AllVariants_Summary(t *testing.T) { type fixtureCase struct { - filename string - symName string - symKind model.NodeKind - label string + filename string + symName string + symKind model.NodeKind + label string } cases := []fixtureCase{ @@ -611,14 +611,14 @@ func TestPythonDocstring_AllVariants_Summary(t *testing.T) { } type row struct { - label string - nodeType string - parentChain string - lines string - prefix string - quoteStyle string - inComment bool - intentBound bool + label string + nodeType string + parentChain string + lines string + prefix string + quoteStyle string + inComment bool + intentBound bool } var rows []row @@ -658,10 +658,10 @@ func TestPythonDocstring_AllVariants_Summary(t *testing.T) { r.lines = "-" } - // 바인딩 확인 - b := parse.NewBinder() - binderComments := binderFromWalkerComments(walkerComments) - bindings := b.Bind(binderComments, nodes, "python", strings.Split(string(content), "\n")) + // 바인딩 확인 + b := parse.NewBinder() + binderComments := binderFromWalkerComments(walkerComments) + bindings := b.Bind(binderComments, nodes, "python", strings.Split(string(content), "\n")) for _, binding := range bindings { var match bool if tc.symName == "" { @@ -732,10 +732,10 @@ func TestPythonDocstring_AllVariants_Summary(t *testing.T) { // (구현 수정 없이, 단순히 gap 계산 로직을 이해하기 위한 측정) func TestPythonDocstring_GapAnalysis(t *testing.T) { type gapCase struct { - name string - filename string - symName string - symKind model.NodeKind + name string + filename string + symName string + symKind model.NodeKind // docstring이 CommentBlock으로 주입될 때 예상 StartLine, EndLine fakeCommentStart int fakeCommentEnd int diff --git a/internal/pathutil/exclude_test.go b/internal/pathutil/exclude_test.go index 93f5333..329dc1f 100644 --- a/internal/pathutil/exclude_test.go +++ b/internal/pathutil/exclude_test.go @@ -73,7 +73,7 @@ func TestLoadIncludePathsFromConfig_InvalidYAMLReturnsError(t *testing.T) { func TestMatchIncludePaths(t *testing.T) { tests := []struct { name string - relPath string + relPath string includePaths []string want bool }{ diff --git a/internal/service/indexer.go b/internal/service/indexer.go index 6c4c6da..343b2e1 100644 --- a/internal/service/indexer.go +++ b/internal/service/indexer.go @@ -135,12 +135,6 @@ func newParsedBuildNodeBatch(relPath string, content []byte, nodes []model.Node, return out } -// newParsedBuildEdgeBatch wraps parsed edges so they can be persisted after their owning nodes. -// @intent defer edge upserts until referenced nodes exist in the graph store. -func newParsedBuildEdgeBatch(relPath string, edges []model.Edge) parsedBuildEdgeBatch { - return parsedBuildEdgeBatch{relPath: relPath, edges: edges} -} - // bindAndReleaseNodeBatch upserts a parsed file's nodes and binds its comment annotations within a transaction-scoped store. // @intent persist nodes and their annotation bindings atomically per file before releasing comment buffers. // @sideEffect writes graph nodes and annotation rows via the transaction-scoped store. @@ -1290,18 +1284,6 @@ func affectedUpdateFiles(currentHashes map[string]string, existingNodesByFile ma return files } -// existingFilesMissingFrom returns paths that were previously stored but are no longer present on disk. -// @intent identify deletions so the incremental syncer can remove their nodes and edges. -func existingFilesMissingFrom(files map[string]incremental.FileInfo, existingFiles []string) []string { - deleted := make([]string, 0) - for _, fp := range existingFiles { - if _, ok := files[fp]; !ok { - deleted = append(deleted, fp) - } - } - return deleted -} - // @intent detect deleted paths from the spool snapshot without rebuilding a full in-memory FileInfo map. func existingFilesMissingFromSet(currentFiles map[string]struct{}, existingFiles []string) []string { deleted := make([]string, 0) @@ -1441,14 +1423,6 @@ func (s *GraphService) parserForExt(ext string) (Parser, bool) { return parser, ok } -// @intent rebuild the namespace-scoped search index after graph mutations when a search backend is configured. -func (s *GraphService) rebuildSearch(ctx context.Context) error { - if s.SearchBackend == nil || s.DB == nil { - return nil - } - return s.rebuildSearchWithDB(ctx, s.DB) -} - // rebuildSearchNodes refreshes search documents for the given node IDs when a search backend is configured. // @intent perform incremental search index updates after changed-node sets are known. func (s *GraphService) rebuildSearchNodes(ctx context.Context, nodeIDs []uint) error { @@ -1654,21 +1628,6 @@ func mergeLanguagePackages(dst map[string]languagePackageInfo, ambiguous map[str } } -// importPackageNames extracts a mapping of import paths to package names. -// @intent allow the edge resolver to look up package names by their source import paths. -func importPackageNames(packages map[string]languagePackageInfo) map[string]string { - if len(packages) == 0 { - return nil - } - names := make(map[string]string, len(packages)) - for importPath, pkg := range packages { - if importPath != "" && pkg.Name != "" { - names[importPath] = pkg.Name - } - } - return names -} - // @intent normalize discovered package imports into the canonical names used when resolving cross-file imports during parsing. func importPackageContext(packages map[string]languagePackageInfo) map[string]string { if len(packages) == 0 { From 836b70159b47c25f0aec8b735510a8eb8a38ae05 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:41:26 +0900 Subject: [PATCH 17/18] Only classify test-named functions as test nodes mapDefTypeToNodeKind marked any declaration whose name started with the language test prefix as a test node, so a production type TestConfig or a function testimonialCard (prefix "test" + lowercase continuation) was miscategorized, which then excluded it from dead-code analysis and changed its search kind. Restrict the test check to function/method declarations and require a word boundary after a bare-word prefix (separator-terminated prefixes like "test_" already encode the boundary). Co-Authored-By: Claude Fable 5 --- internal/parse/treesitter/walker.go | 38 +++++++++++++---- internal/parse/treesitter/walker_test.go | 52 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/internal/parse/treesitter/walker.go b/internal/parse/treesitter/walker.go index ed7dd52..97cb165 100644 --- a/internal/parse/treesitter/walker.go +++ b/internal/parse/treesitter/walker.go @@ -656,24 +656,48 @@ func (w *Walker) inferEnclosingReceiver(defNode *sitter.Node, content []byte) st // mapDefTypeToNodeKind maps query definition labels to internal node kinds. // @intent keep language query captures aligned with graph node categorization -// @domainRule declarations matching the configured test prefix become test nodes +// @domainRule only function/method declarations whose name is a test name become test nodes; +// types, classes, and interfaces are never tests even when their name starts with the prefix func (w *Walker) mapDefTypeToNodeKind(defType string, name string) model.NodeKind { - if w.spec.TestPrefix != "" && strings.HasPrefix(name, w.spec.TestPrefix) { - return model.NodeKindTest - } - switch defType { case "class": return model.NodeKindClass case "interface", "type": return model.NodeKindType - case "function", "method": - return model.NodeKindFunction default: + if isTestName(name, w.spec.TestPrefix) { + return model.NodeKindTest + } return model.NodeKindFunction } } +// isTestName reports whether name denotes a test given the language's test prefix. +// @intent avoid misclassifying production symbols like testimonialCard or TestConfig as tests. +// @domainRule a separator-terminated prefix (e.g. "test_") is a boundary on its own; a bare-word +// prefix (e.g. "Test", "test", "TEST") must be followed by end-of-name or a non-lowercase character +// so it does not swallow a longer lowercase word. +func isTestName(name, prefix string) bool { + if prefix == "" || !strings.HasPrefix(name, prefix) { + return false + } + last := prefix[len(prefix)-1] + if !isASCIILetterOrDigit(last) { + return true + } + rest := name[len(prefix):] + if rest == "" { + return true + } + c := rest[0] + return c < 'a' || c > 'z' +} + +// @intent classify ASCII identifier characters for test-prefix boundary detection. +func isASCIILetterOrDigit(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + // buildQualifiedName joins package, receiver, and declaration name into a stable identifier. // @intent generate graph keys that distinguish methods from package-level declarations func (w *Walker) buildQualifiedName(pkg, receiver, name string) string { diff --git a/internal/parse/treesitter/walker_test.go b/internal/parse/treesitter/walker_test.go index b5e229c..1dc7980 100644 --- a/internal/parse/treesitter/walker_test.go +++ b/internal/parse/treesitter/walker_test.go @@ -1681,6 +1681,58 @@ func TestAdd(t *testing.T) {} } } +func TestParseGo_TestPrefixDoesNotMisclassifyProductionDecls(t *testing.T) { + src := `package main + +type TestConfig struct{} + +func Testimony() {} + +func TestAdd(t *testing.T) {} +` + w := NewWalker(GoSpec) + nodes, _, err := w.Parse("main_test.go", []byte(src)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + testNodes := filterByKind(nodes, model.NodeKindTest) + if len(testNodes) != 1 || testNodes[0].Name != "TestAdd" { + t.Fatalf("expected only TestAdd as a test node, got %+v", testNodes) + } + // TestConfig (a type) and Testimony (Test + lowercase 'i') must not be tests. + byName := map[string]model.NodeKind{} + for _, n := range nodes { + byName[n.Name] = n.Kind + } + if byName["TestConfig"] == model.NodeKindTest { + t.Errorf("TestConfig type must not be a test node") + } + if byName["Testimony"] == model.NodeKindTest { + t.Errorf("Testimony (lowercase continuation) must not be a test node") + } +} + +func TestParseTypeScript_TestPrefixRequiresWordBoundary(t *testing.T) { + src := `function testimonialCard() {} +function testLogin() {} +` + w := NewWalker(TypeScriptSpec) + nodes, _, err := w.Parse("app.ts", []byte(src)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + kinds := map[string]model.NodeKind{} + for _, n := range nodes { + kinds[n.Name] = n.Kind + } + if kinds["testimonialCard"] == model.NodeKindTest { + t.Errorf("testimonialCard must not be classified as a test") + } + if kinds["testLogin"] != model.NodeKindTest { + t.Errorf("testLogin should be classified as a test, got %v", kinds["testLogin"]) + } +} + func TestParsePython_TestFunction(t *testing.T) { src := `def test_add(): assert 1 + 1 == 2 From cb7bcd8a2c21c8df6758e7efde42fe7049699267 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Fri, 10 Jul 2026 22:41:26 +0900 Subject: [PATCH 18/18] Fix non-transactional incremental update delete protocol updateGraphWithoutTx (the path taken by the MCP build_or_update_graph / parse_project tools, whose injected syncer is not transactional) passed the full existing-file set to the first normal batch. Because SyncWithExisting deletes existing files absent from the batch, a multi-record spool deleted files belonging to later batches and re-added them (churning node IDs, annotations, and stats); it also double-counted deletions against the explicit delete pass, and skipped deletions entirely when no normal batch ran. Mirror the transactional path: normal batches carry no existing files and deletions run once, unconditionally. Co-Authored-By: Claude Fable 5 --- internal/mcp/handlers_test.go | 53 ++++++++++++++------------ internal/service/indexer.go | 34 +++-------------- internal/service/indexer_test.go | 65 +++++++++++++++++++++++++++----- 3 files changed, 91 insertions(+), 61 deletions(-) diff --git a/internal/mcp/handlers_test.go b/internal/mcp/handlers_test.go index 6272e2c..423be8d 100644 --- a/internal/mcp/handlers_test.go +++ b/internal/mcp/handlers_test.go @@ -973,14 +973,12 @@ func TestBuildOrUpdateGraph_IncrementalIncludePaths_DefaultsToReplace(t *testing mockSync := &mockIncrementalSyncer{result: &incremental.SyncStats{}} deps.Incremental = mockSync + // Only handler.go is on disk; the out-of-scope other.go is missing. Under default replace + // semantics the delete pass must consider all namespace files, so other.go is deletable. dir := t.TempDir() os.MkdirAll(filepath.Join(dir, "src", "api"), 0o755) - os.MkdirAll(filepath.Join(dir, "src", "other"), 0o755) writeGoFile(t, filepath.Join(dir, "src", "api"), "handler.go", `package api func Handler() {} -`) - writeGoFile(t, filepath.Join(dir, "src", "other"), "other.go", `package other -func Other() {} `) callTool(t, deps, "build_or_update_graph", map[string]any{ @@ -994,16 +992,24 @@ func Other() {} if !mockSync.syncWithExisting { t.Fatal("expected Incremental.SyncWithExisting to be called") } - if len(mockSync.existingCalls) == 0 { - t.Fatal("expected at least one SyncWithExisting call") + deleteExisting := deletePassExistingFiles(mockSync) + if deleteExisting == nil { + t.Fatalf("expected a delete pass for the missing out-of-scope file, calls=%v", mockSync.existingCalls) } - firstExisting := mockSync.existingCalls[0] - if len(firstExisting) != 2 { - t.Fatalf("expected default replace semantics to pass all namespace files, got %v", firstExisting) + if !containsStringInSlice(deleteExisting, filepath.Join("src", "other", "other.go")) { + t.Fatalf("default replace must consider the out-of-scope file for deletion, got %v", deleteExisting) } - if !containsStringInSlice(firstExisting, filepath.Join("src", "other", "other.go")) { - t.Fatalf("expected existingFiles to include out-of-scope file under default replace semantics, got %v", firstExisting) +} + +// deletePassExistingFiles returns the existingFiles of the delete pass (the SyncWithExisting +// call that received no files), or nil when no delete pass ran. +func deletePassExistingFiles(m *mockIncrementalSyncer) []string { + for i := range m.filesCalls { + if len(m.filesCalls[i]) == 0 && len(m.existingCalls[i]) > 0 { + return m.existingCalls[i] + } } + return nil } func TestBuildOrUpdateGraph_IncrementalIncludePaths_ReplaceFalsePreservesOutOfScopeFiles(t *testing.T) { @@ -1020,14 +1026,13 @@ func TestBuildOrUpdateGraph_IncrementalIncludePaths_ReplaceFalsePreservesOutOfSc mockSync := &mockIncrementalSyncer{result: &incremental.SyncStats{}} deps.Incremental = mockSync + // The seeded handler.go and other.go are both missing from disk; a new in-scope file is + // added. With replace=false + include_paths, only the in-scope handler.go may be deleted, + // and the out-of-scope other.go must be scoped out of the existing-file set entirely. dir := t.TempDir() os.MkdirAll(filepath.Join(dir, "src", "api"), 0o755) - os.MkdirAll(filepath.Join(dir, "src", "other"), 0o755) - writeGoFile(t, filepath.Join(dir, "src", "api"), "handler.go", `package api -func Handler() {} -`) - writeGoFile(t, filepath.Join(dir, "src", "other"), "other.go", `package other -func Other() {} + writeGoFile(t, filepath.Join(dir, "src", "api"), "new.go", `package api +func New() {} `) callTool(t, deps, "build_or_update_graph", map[string]any{ @@ -1042,15 +1047,15 @@ func Other() {} if !mockSync.syncWithExisting { t.Fatal("expected Incremental.SyncWithExisting to be called") } - if len(mockSync.existingCalls) == 0 { - t.Fatal("expected at least one SyncWithExisting call") + deleteExisting := deletePassExistingFiles(mockSync) + if deleteExisting == nil { + t.Fatalf("expected a delete pass for the missing in-scope file, calls=%v", mockSync.existingCalls) } - firstExisting := mockSync.existingCalls[0] - if containsStringInSlice(firstExisting, filepath.Join("src", "other", "other.go")) { - t.Fatalf("expected replace=false to exclude out-of-scope file from existingFiles, got %v", firstExisting) + if containsStringInSlice(deleteExisting, filepath.Join("src", "other", "other.go")) { + t.Fatalf("replace=false must exclude the out-of-scope file from deletions, got %v", deleteExisting) } - if !containsStringInSlice(firstExisting, filepath.Join("src", "api", "handler.go")) { - t.Fatalf("expected replace=false to keep in-scope file, got %v", firstExisting) + if !containsStringInSlice(deleteExisting, filepath.Join("src", "api", "handler.go")) { + t.Fatalf("replace=false must still delete the missing in-scope file, got %v", deleteExisting) } } diff --git a/internal/service/indexer.go b/internal/service/indexer.go index 343b2e1..8340af9 100644 --- a/internal/service/indexer.go +++ b/internal/service/indexer.go @@ -1204,8 +1204,10 @@ func (s *GraphService) updateGraphWithoutTx(ctx context.Context, absDir string, } stats := &incremental.SyncStats{} - normalExistingFiles := existingFilesExcluding(existingFiles, forceFiles) - passedExistingFiles := false + // Normal batches must never carry existingFiles: SyncWithExisting deletes existing files + // absent from the batch, so a multi-record spool would delete files belonging to later + // batches (then re-add them, churning node IDs and stats). Deletions are handled once, + // explicitly, below — mirroring the transactional path (applyUpdateSpoolInTx). for _, path := range spool.records { record, err := spool.readRecord(path) if err != nil { @@ -1215,19 +1217,14 @@ func (s *GraphService) updateGraphWithoutTx(ctx context.Context, absDir string, if len(normalFiles) == 0 { continue } - batchExistingFiles := []string(nil) - if !passedExistingFiles { - batchExistingFiles = normalExistingFiles - passedExistingFiles = true - } - batchStats, err := opts.Syncer.SyncWithExisting(ctx, normalFiles, batchExistingFiles) + batchStats, err := opts.Syncer.SyncWithExisting(ctx, normalFiles, nil) if err != nil { return nil, trace.Wrap(err, "incremental sync") } addSyncStats(stats, batchStats) } deletedFiles := existingFilesMissingFromSet(spool.currentFiles, existingFiles) - if len(deletedFiles) > 0 && passedExistingFiles { + if len(deletedFiles) > 0 { batchStats, err := opts.Syncer.SyncWithExisting(ctx, nil, deletedFiles) if err != nil { return nil, trace.Wrap(err, "incremental delete sync") @@ -1295,25 +1292,6 @@ func existingFilesMissingFromSet(currentFiles map[string]struct{}, existingFiles return deleted } -// existingFilesExcluding filters out paths excluded from the current sync pass. -// @intent keep incremental sync from revisiting files the caller already ruled out. -func existingFilesExcluding(existingFiles []string, exclude map[string]struct{}) []string { - if len(existingFiles) == 0 { - return nil - } - if len(exclude) == 0 { - return append([]string(nil), existingFiles...) - } - filtered := make([]string, 0, len(existingFiles)) - for _, fp := range existingFiles { - if _, ok := exclude[fp]; ok { - continue - } - filtered = append(filtered, fp) - } - return filtered -} - // affectedNodeIDsForUpdate collects node IDs whose search documents must be refreshed for a given change set. // @intent merge previously stored node IDs with newly created ones so the search index sees both removals and additions. func affectedNodeIDsForUpdate(ctx context.Context, db *gorm.DB, existingNodesByFile map[string][]model.Node, changedFiles, deletedFiles []string) ([]uint, error) { diff --git a/internal/service/indexer_test.go b/internal/service/indexer_test.go index 48e0d04..ce85c8b 100644 --- a/internal/service/indexer_test.go +++ b/internal/service/indexer_test.go @@ -2701,6 +2701,40 @@ func TestUpdate_MaxTotalParsedBytesRejectsBeforeSync(t *testing.T) { } } +func TestUpdateGraphWithoutTx_DeletesEvenWithNoNormalBatches(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + if err != nil { + t.Fatalf("open db: %v", err) + } + if err := db.AutoMigrate(&model.Node{}); err != nil { + t.Fatalf("migrate nodes: %v", err) + } + if err := db.Create(&model.Node{Namespace: ctxns.DefaultNamespace, FilePath: "gone.go"}).Error; err != nil { + t.Fatalf("seed node: %v", err) + } + + svc := &GraphService{ + DB: db, + Walkers: map[string]*treesitter.Walker{".go": treesitter.NewWalker(treesitter.GoSpec)}, + Logger: slog.Default(), + } + + // Empty working dir: nothing to parse, so there are no normal batches. The previously + // stored gone.go must still be deleted (regression: the delete pass was gated on a normal + // batch having run). + tmpDir := t.TempDir() + syncer := &recordingIncrementalSyncer{result: &incremental.SyncStats{}} + if _, err := svc.Update(context.Background(), UpdateOptions{BuildOptions: BuildOptions{Dir: tmpDir, SkipSearchRebuild: true}, Syncer: syncer}); err != nil { + t.Fatalf("Update: %v", err) + } + if len(syncer.calls) != 1 { + t.Fatalf("expected a single delete pass, got %d calls: %v", len(syncer.calls), syncer.calls) + } + if got, want := syncer.calls[0].existingFiles, []string{"gone.go"}; !reflect.DeepEqual(got, want) { + t.Fatalf("delete pass existingFiles = %v, want %v", got, want) + } +} + func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) { db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { @@ -2709,6 +2743,8 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if err := db.AutoMigrate(&model.Node{}); err != nil { t.Fatalf("migrate nodes: %v", err) } + // Both seeded files are missing from disk. handler.go is inside the include path and must be + // deleted; helper.go is outside it and must be scoped out of the existing-file set entirely. if err := db.Create(&model.Node{Namespace: ctxns.DefaultNamespace, FilePath: filepath.Join("src", "api", "handler.go")}).Error; err != nil { t.Fatalf("seed api node: %v", err) } @@ -2727,8 +2763,10 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if err := os.MkdirAll(apiDir, 0o755); err != nil { t.Fatalf("mkdir api: %v", err) } - if err := os.WriteFile(filepath.Join(apiDir, "handler.go"), []byte("package api\n\nfunc Handler() {}\n"), 0o644); err != nil { - t.Fatalf("write handler: %v", err) + // A new file under the include path so the update has something to parse; the seeded + // handler.go is absent from disk and should be detected as a deletion. + if err := os.WriteFile(filepath.Join(apiDir, "new.go"), []byte("package api\n\nfunc New() {}\n"), 0o644); err != nil { + t.Fatalf("write new: %v", err) } syncer := &recordingIncrementalSyncer{result: &incremental.SyncStats{}} @@ -2736,8 +2774,18 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if err != nil { t.Fatalf("Update: %v", err) } - if got, want := syncer.existingFiles, []string{filepath.Join("src", "api", "handler.go")}; !reflect.DeepEqual(got, want) { - t.Fatalf("existingFiles mismatch: got=%v want=%v", got, want) + + var deleteCall *recordingSyncCall + for i := range syncer.calls { + if len(syncer.calls[i].files) == 0 && len(syncer.calls[i].existingFiles) > 0 { + deleteCall = &syncer.calls[i] + } + } + if deleteCall == nil { + t.Fatalf("expected a delete pass for the missing include-path file, calls=%v", syncer.calls) + } + if got, want := deleteCall.existingFiles, []string{filepath.Join("src", "api", "handler.go")}; !reflect.DeepEqual(got, want) { + t.Fatalf("delete pass existingFiles mismatch: got=%v want=%v (helper.go outside include path must be scoped out)", got, want) } } @@ -3045,11 +3093,10 @@ func TestUpdateGraphWithoutTx_DoesNotDeleteForcedFilesDuringNormalSync(t *testin if _, ok := first.files["target.go"]; !ok { t.Fatalf("expected changed target.go in normal sync files, got %v", sortedIncrementalFileKeys(first.files)) } - if slices.Contains(first.existingFiles, "source.go") { - t.Fatalf("expected forced source.go to be excluded from normal sync existingFiles, got %v", first.existingFiles) - } - if !slices.Contains(first.existingFiles, "deleted.go") { - t.Fatalf("expected deleted.go to remain in normal sync existingFiles, got %v", first.existingFiles) + // Normal sync passes no existingFiles: deletions are handled by the separate delete pass, + // so a normal batch can never delete files that belong to other batches (mirrors the tx path). + if len(first.existingFiles) != 0 { + t.Fatalf("expected normal sync to pass no existingFiles, got %v", first.existingFiles) } second := syncer.calls[1] if len(second.files) != 0 {