diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2381867..249da96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: - name: Run go vet run: make vet + - name: Run go test + run: CGO_ENABLED=1 go test -tags "fts5" ./... -count=1 + - name: Build release binaries run: make build diff --git a/.gitignore b/.gitignore index 55af86f..84c89eb 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ pyproject.toml /web/wiki/dist/ /web/wiki/node_modules/ search-retrieval.md +_workspace/ diff --git a/cmd/ccg/main_postgres_test.go b/cmd/ccg/main_postgres_test.go index b1883f1..4d088b5 100644 --- a/cmd/ccg/main_postgres_test.go +++ b/cmd/ccg/main_postgres_test.go @@ -157,7 +157,7 @@ func TestRunMigrations_PostgresDownRestoresNullableColumns(t *testing.T) { t.Fatalf("run down migration: %v", err) } - var version migrateSchemaVersion + var version migration.MigrationSchemaVersion if err := db.Table("schema_migrations").First(&version).Error; err != nil { t.Fatalf("load schema version: %v", err) } @@ -190,7 +190,7 @@ func TestRunMigrations_PostgresDownFromVersionThreeDropsPolicyTables(t *testing. t.Fatalf("run down migration: %v", err) } - var version migrateSchemaVersion + var version migration.MigrationSchemaVersion if err := db.Table("schema_migrations").First(&version).Error; err != nil { t.Fatalf("load schema version: %v", err) } diff --git a/docker-compose.langfuse.yaml b/docker-compose.langfuse.yaml deleted file mode 100644 index d8778f5..0000000 --- a/docker-compose.langfuse.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# Make sure to update the credential placeholders with your own secrets. -# We mark them with # CHANGEME in the file below. -# In addition, we recommend to restrict inbound traffic on the host to langfuse-web (port 3000) and minio (port 9090) only. -# All other components are bound to localhost (127.0.0.1) to only accept connections from the local machine. -# External connections from other machines will not be able to reach these services directly. -services: - langfuse-worker: - image: docker.io/langfuse/langfuse-worker:3 - restart: always - depends_on: &langfuse-depends-on - postgres: - condition: service_healthy - minio: - condition: service_healthy - redis: - condition: service_healthy - clickhouse: - condition: service_healthy - ports: - - 127.0.0.1:3030:3030 - environment: &langfuse-worker-env - NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000} - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres} # CHANGEME - SALT: ${SALT:-mysalt} # CHANGEME - ENCRYPTION_KEY: ${ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} # CHANGEME: generate via `openssl rand -hex 32` - TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true} - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false} - CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000} - CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123} - CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse} - CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME - CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false} - LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false} - LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false} - LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity} - LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse} - LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto} - LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio} - LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME - LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000} - LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true} - LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/} - LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse} - LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto} - LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio} - LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME - LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://localhost:9090} - LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true} - LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/} - LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false} - LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${LANGFUSE_S3_BATCH_EXPORT_BUCKET:-langfuse} - LANGFUSE_S3_BATCH_EXPORT_PREFIX: ${LANGFUSE_S3_BATCH_EXPORT_PREFIX:-exports/} - LANGFUSE_S3_BATCH_EXPORT_REGION: ${LANGFUSE_S3_BATCH_EXPORT_REGION:-auto} - LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_ENDPOINT:-http://minio:9000} - LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT:-http://localhost:9090} - LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: ${LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID:-minio} - LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: ${LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME - LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: ${LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE:-true} - LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-} - LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-} - REDIS_HOST: ${REDIS_HOST:-redis} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_AUTH: ${REDIS_AUTH:-myredissecret} # CHANGEME - REDIS_TLS_ENABLED: ${REDIS_TLS_ENABLED:-false} - REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt} - REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt} - REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key} - EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-} - SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-} - - langfuse-web: - image: docker.io/langfuse/langfuse:3 - restart: always - depends_on: *langfuse-depends-on - ports: - - 3000:3000 - environment: - <<: *langfuse-worker-env - NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME - LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-} - LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-} - LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-} - LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-} - LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-} - LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-} - LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-} - LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-} - LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-} - - clickhouse: - image: docker.io/clickhouse/clickhouse-server - restart: always - user: "101:101" - environment: - CLICKHOUSE_DB: default - CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse} - CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME - volumes: - - langfuse_clickhouse_data:/var/lib/clickhouse - - langfuse_clickhouse_logs:/var/log/clickhouse-server - ports: - - 127.0.0.1:8123:8123 - - 127.0.0.1:9000:9000 - healthcheck: - test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1 - interval: 5s - timeout: 5s - retries: 10 - start_period: 1s - - minio: - image: cgr.dev/chainguard/minio - restart: always - entrypoint: sh - # create the 'langfuse' bucket before starting the service - command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" - --console-address ":9001" /data' - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret} # CHANGEME - ports: - - 9090:9000 - - 127.0.0.1:9091:9001 - volumes: - - langfuse_minio_data:/data - healthcheck: - test: [ "CMD", "mc", "ready", "local" ] - interval: 1s - timeout: 5s - retries: 5 - start_period: 1s - - redis: - image: docker.io/redis:7 - restart: always - # CHANGEME: row below to secure redis password - command: > - --requirepass ${REDIS_AUTH:-myredissecret} --maxmemory-policy noeviction - ports: - - 127.0.0.1:6379:6379 - volumes: - - langfuse_redis_data:/data - healthcheck: - test: [ "CMD", "redis-cli", "ping" ] - interval: 3s - timeout: 10s - retries: 10 - - postgres: - image: docker.io/postgres:${POSTGRES_VERSION:-17} - restart: always - healthcheck: - test: [ "CMD-SHELL", "pg_isready -U postgres" ] - interval: 3s - timeout: 3s - retries: 10 - environment: - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} # CHANGEME - POSTGRES_DB: ${POSTGRES_DB:-postgres} - TZ: UTC - PGTZ: UTC - ports: - - 127.0.0.1:5432:5432 - volumes: - - langfuse_postgres_data:/var/lib/postgresql/data - -volumes: - langfuse_postgres_data: - driver: local - langfuse_clickhouse_data: - driver: local - langfuse_clickhouse_logs: - driver: local - langfuse_minio_data: - driver: local - langfuse_redis_data: - driver: local diff --git a/internal/analysis/changes/git.go b/internal/analysis/changes/git.go index 6006085..e4d1b11 100644 --- a/internal/analysis/changes/git.go +++ b/internal/analysis/changes/git.go @@ -37,7 +37,10 @@ func NewExecGitClient() *ExecGitClient { // @requires repoDir points to a valid git working tree // @ensures returned file paths are trimmed and exclude blank lines func (g *ExecGitClient) ChangedFiles(ctx context.Context, repoDir, baseRef string) ([]string, error) { - out, err := runGitLimited(ctx, repoDir, []string{"diff", "--name-only", baseRef}) + if err := validateBaseRef(baseRef); err != nil { + return nil, err + } + out, err := runGitLimited(ctx, repoDir, []string{"-c", "core.quotePath=false", "diff", "--name-only", baseRef, "--"}) if err != nil { return nil, trace.Wrap(err, "git diff --name-only") } @@ -66,7 +69,10 @@ func (g *ExecGitClient) ChangedFiles(ctx context.Context, repoDir, baseRef strin // @requires repoDir points to a valid git working tree // @ensures each returned hunk has a file path and inclusive start/end lines func (g *ExecGitClient) DiffHunks(ctx context.Context, repoDir, baseRef string, paths []string) ([]Hunk, error) { - args := []string{"diff", "-U0", baseRef, "--"} + if err := validateBaseRef(baseRef); err != nil { + return nil, err + } + args := []string{"-c", "core.quotePath=false", "diff", "-U0", baseRef, "--"} args = append(args, paths...) out, err := runGitLimited(ctx, repoDir, args) if err != nil { @@ -98,6 +104,19 @@ func (g *ExecGitClient) DiffHunks(ctx context.Context, repoDir, baseRef string, return hunks, nil } +// validateBaseRef rejects base revisions that git would parse as command options. +// @intent block git argument injection through the caller-supplied base ref, which sits +// before the "--" separator and would otherwise be interpreted as a diff flag. +func validateBaseRef(baseRef string) error { + if baseRef == "" { + return fmt.Errorf("base ref must not be empty") + } + if strings.HasPrefix(baseRef, "-") { + return fmt.Errorf("invalid base ref %q: must not start with '-'", baseRef) + } + return nil +} + // runGitLimited runs a git subcommand with the default output size cap. // @intent share a single bounded git invocation helper across diff operations func runGitLimited(ctx context.Context, repoDir string, args []string) ([]byte, error) { diff --git a/internal/analysis/changes/git_test.go b/internal/analysis/changes/git_test.go index 8aabf72..cdf5870 100644 --- a/internal/analysis/changes/git_test.go +++ b/internal/analysis/changes/git_test.go @@ -29,6 +29,46 @@ func TestGitClient_ChangedFiles(t *testing.T) { } } +func TestGitClient_RejectsOptionLikeBaseRef(t *testing.T) { + dir := initTestRepo(t) + writeFile(t, dir, "hello.go", "package main\n") + gitCommit(t, dir, "initial") + + git := NewExecGitClient() + for _, base := range []string{"--output=/tmp/pwned", "-U9999", "--no-index"} { + if _, err := git.ChangedFiles(context.Background(), dir, base); err == nil { + t.Errorf("ChangedFiles accepted option-like base %q", base) + } + if _, err := git.DiffHunks(context.Background(), dir, base, []string{"hello.go"}); err == nil { + t.Errorf("DiffHunks accepted option-like base %q", base) + } + } +} + +func TestGitClient_ChangedFiles_NonASCIIPath(t *testing.T) { + dir := initTestRepo(t) + writeFile(t, dir, "한글.go", "package main\n") + gitCommit(t, dir, "initial") + writeFile(t, dir, "한글.go", "package main\n\nfunc Hello() {}\n") + + git := NewExecGitClient() + files, err := git.ChangedFiles(context.Background(), dir, "HEAD") + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || files[0] != "한글.go" { + t.Fatalf("expected unquoted 한글.go, got %v", files) + } + + hunks, err := git.DiffHunks(context.Background(), dir, "HEAD", []string{"한글.go"}) + if err != nil { + t.Fatal(err) + } + if len(hunks) == 0 || hunks[0].FilePath != "한글.go" { + t.Fatalf("expected hunk for unquoted 한글.go, got %+v", hunks) + } +} + func TestGitClient_DiffHunks(t *testing.T) { dir := initTestRepo(t) writeFile(t, dir, "hello.go", "package main\n\nfunc Old() {}\n") diff --git a/internal/analysis/fallback/service.go b/internal/analysis/fallback/service.go index da23a3e..523f6c1 100644 --- a/internal/analysis/fallback/service.go +++ b/internal/analysis/fallback/service.go @@ -86,13 +86,17 @@ func (s *Service) FindSuspectsPage(ctx context.Context, opts Options) (Result, e results := make([]SuspectEdge, 0, len(edges)) for _, edge := range edges { source, err := s.store.GetNodeByID(ctx, edge.FromNodeID) - if err != nil || source == nil { + if err != nil { return Result{}, err } target, err := s.store.GetNodeByID(ctx, edge.ToNodeID) - if err != nil || target == nil { + if err != nil { return Result{}, err } + // A dangling edge (endpoint node missing) must not silently drop the whole page. + if source == nil || target == nil { + continue + } sourceAnn, err := s.store.GetAnnotation(ctx, source.ID) if err != nil { return Result{}, err diff --git a/internal/analysis/fallback/service_test.go b/internal/analysis/fallback/service_test.go index 94dbd42..f62d70a 100644 --- a/internal/analysis/fallback/service_test.go +++ b/internal/analysis/fallback/service_test.go @@ -63,6 +63,44 @@ func TestSuspectFallbackEdges_FlagsDisjointIntentAndDomainRule(t *testing.T) { } } +func TestSuspectFallbackEdges_SkipsEdgeWithMissingEndpointInsteadOfDroppingPage(t *testing.T) { + db := setupFallbackDB(t) + ctx := context.Background() + store := gormstore.New(db) + + source := model.Node{QualifiedName: "pkg.A", Kind: model.NodeKindFunction, Name: "A", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"} + target := model.Node{QualifiedName: "pkg.B", Kind: model.NodeKindFunction, Name: "B", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"} + if err := store.UpsertNodes(ctx, []model.Node{source, target}); err != nil { + t.Fatal(err) + } + sourceNode, _ := store.GetNode(ctx, "pkg.A") + targetNode, _ := store.GetNode(ctx, "pkg.B") + if err := store.UpsertEdges(ctx, []model.Edge{ + // dangling edge: endpoint node id does not exist + {FromNodeID: 99999, ToNodeID: targetNode.ID, Kind: model.EdgeKindFallbackCalls, Fingerprint: "a-dangling"}, + {FromNodeID: sourceNode.ID, ToNodeID: targetNode.ID, Kind: model.EdgeKindFallbackCalls, Fingerprint: "b-valid"}, + }); err != nil { + t.Fatal(err) + } + if err := store.UpsertAnnotation(ctx, &model.Annotation{NodeID: sourceNode.ID, Summary: "auth", Tags: []model.DocTag{{Kind: model.TagIntent, Value: "verify credentials", Ordinal: 0}}}); err != nil { + t.Fatal(err) + } + if err := store.UpsertAnnotation(ctx, &model.Annotation{NodeID: targetNode.ID, Summary: "billing", Tags: []model.DocTag{{Kind: model.TagIntent, Value: "render invoice", Ordinal: 0}}}); err != nil { + t.Fatal(err) + } + + results, err := New(db, store).FindSuspects(ctx, Options{}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("dangling edge must be skipped, not drop the whole page: got %d results", len(results)) + } + if results[0].Source.QualifiedName != "pkg.A" { + t.Fatalf("expected surviving valid edge, got %+v", results[0]) + } +} + func TestSuspectFallbackEdges_IgnoresOverlappingAnnotationContext(t *testing.T) { db := setupFallbackDB(t) ctx := context.Background() diff --git a/internal/analysis/flows/flows.go b/internal/analysis/flows/flows.go index ffa50d4..1e569a7 100644 --- a/internal/analysis/flows/flows.go +++ b/internal/analysis/flows/flows.go @@ -143,7 +143,7 @@ func (t *Tracer) TraceFlowBounded(ctx context.Context, startNodeID uint, opts Tr Truncated: truncated, MaxNodes: opts.MaxNodes, ReturnedNodes: len(members), - ContainsFallbackCalls: containsFallbackCalls, + ContainsFallbackCalls: containsFallbackCalls, FallbackEdgesCount: fallbackEdges, }, nil } diff --git a/internal/annotation/normalizer.go b/internal/annotation/normalizer.go index ac0a8a0..7dce1f4 100644 --- a/internal/annotation/normalizer.go +++ b/internal/annotation/normalizer.go @@ -84,7 +84,7 @@ func stripBlockDelimiters(text string, language string) string { } // stripPythonDocstringDelimiters removes triple-quote wrappers from a Python docstring body. -// @intent expose the raw docstring text by trying both """ and ''' triple-quote forms. +// @intent expose the raw docstring text by trying both """ and ”' triple-quote forms. func stripPythonDocstringDelimiters(text string) (string, bool) { for _, quote := range []string{"\"\"\"", "'''"} { if stripped, ok := stripPythonQuotedString(text, quote); ok { diff --git a/internal/benchmark/analyze.go b/internal/benchmark/analyze.go deleted file mode 100644 index aaf2095..0000000 --- a/internal/benchmark/analyze.go +++ /dev/null @@ -1,233 +0,0 @@ -// @index Match analysis helpers for benchmark run results. -package benchmark - -import ( - "encoding/json" - "strings" -) - -// @intent decode one query_graph result row so benchmark scoring can inspect returned confidence labels. -type queryGraphResultItem struct { - QualifiedName string `json:"qualified_name"` - Confidence string `json:"confidence"` -} - - -// @intent decode query_graph tool payloads before strict/tentative coverage scoring. -type queryGraphToolResponse struct { - Pattern string `json:"pattern"` - Results []queryGraphResultItem `json:"results"` -} - -// @intent carry strict and tentative symbol expectation totals alongside matched counts. -type queryConfidenceCoverage struct { - StrictCount int - TentativeCount int - StrictMatched int - TentativeMatched int -} - -// fileMatches reports whether actual matches expected, supporting both -// exact match and suffix match to handle absolute vs relative path differences. -// e.g. expected "internal/webhook/handler.go" matches actual "/home/user/repo/internal/webhook/handler.go" -// @intent compare expected files against recorded reads without depending on absolute path stability. -func fileMatches(expected, actual string) bool { - if expected == actual { - return true - } - return strings.HasSuffix(actual, "/"+expected) || strings.HasSuffix(actual, "\\"+expected) -} - -// MatchFiles computes the ratio of expected files found in FilesRead or mentioned -// in the Answer text (as a fallback when tool-call data is unavailable). -// Returns 1.0 if no expected files are specified. -// @intent score whether a benchmark run inspected the files the query was expected to touch. -func MatchFiles(result RunResult, query Query) float64 { - if len(query.ExpectedFiles) == 0 { - return 1.0 - } - var matched int - for _, expected := range query.ExpectedFiles { - found := false - for _, actual := range result.FilesRead { - if fileMatches(expected, actual) { - found = true - break - } - } - if !found && result.Answer != "" { - base := expected[strings.LastIndex(expected, "/")+1:] - found = strings.Contains(result.Answer, expected) || strings.Contains(result.Answer, base) - } - if found { - matched++ - } - } - return float64(matched) / float64(len(query.ExpectedFiles)) -} - -// MatchSymbols computes the ratio of expected symbols found in the answer text. -// Returns 1.0 if no expected symbols are specified. -// @intent score whether the final answer mentioned the symbols the query was expected to surface. -func MatchSymbols(result RunResult, query Query) float64 { - if len(query.ExpectedSymbols) == 0 { - return 1.0 - } - var matched int - for _, sym := range query.ExpectedSymbols { - if strings.Contains(result.Answer, sym) { - matched++ - } - } - return float64(matched) / float64(len(query.ExpectedSymbols)) -} - -// CountCcgToolCalls returns the number of tool calls using mcp__ccg__ prefix. -// @intent quantify how much a benchmark run relied on CCG-specific MCP tools. -func CountCcgToolCalls(result RunResult) int { - var count int - for _, tc := range result.ToolCalls { - if strings.HasPrefix(tc.Tool, "mcp__ccg__") { - count++ - } - } - return count -} - -// matchSymbolsFromToolOutputs computes strict/tentative symbol hit ratios from query_graph -// call outputs. Returns 1.0 when no expectation is provided. -// @intent score query_graph evidence separately from free-form answer text so confidence-aware metrics stay grounded in tool output. -func matchSymbolsFromToolOutputs(result RunResult, expectedStrict, expectedTentative []string) queryConfidenceCoverage { - expectedStrictSet := make(map[string]struct{}, len(expectedStrict)) - for _, sym := range expectedStrict { - expectedStrictSet[sym] = struct{}{} - } - expectedTentativeSet := make(map[string]struct{}, len(expectedTentative)) - for _, sym := range expectedTentative { - expectedTentativeSet[sym] = struct{}{} - } - - coverage := queryConfidenceCoverage{ - StrictCount: len(expectedStrict), - TentativeCount: len(expectedTentative), - } - strictMatched := make(map[string]struct{}, len(expectedStrict)) - tentativeMatched := make(map[string]struct{}, len(expectedTentative)) - - for _, tc := range result.ToolCalls { - if tc.Tool != "mcp__ccg__query_graph" || tc.Output == "" { - continue - } - var parsed queryGraphToolResponse - if err := json.Unmarshal([]byte(tc.Output), &parsed); err != nil { - continue - } - if parsed.Pattern != "callers_of" && parsed.Pattern != "callees_of" { - continue - } - for _, item := range parsed.Results { - switch item.Confidence { - case "strict": - if _, ok := expectedStrictSet[item.QualifiedName]; ok { - if _, exists := strictMatched[item.QualifiedName]; exists { - continue - } - coverage.StrictMatched++ - strictMatched[item.QualifiedName] = struct{}{} - } - case "tentative": - if _, ok := expectedTentativeSet[item.QualifiedName]; ok { - if _, exists := tentativeMatched[item.QualifiedName]; exists { - continue - } - coverage.TentativeMatched++ - tentativeMatched[item.QualifiedName] = struct{}{} - } - } - } - } - return coverage -} - -// @intent avoid divide-by-zero while keeping empty expectation sets neutral in metric aggregation. -func safeRatio(n, d int) float64 { - if d == 0 { - return 1.0 - } - return float64(n) / float64(d) -} - -// computeLLMStrictBias returns: -// - 1.0 when strict expectations are all met or no strict expectation exists. -// - 0.5 when strict is mentioned but tentative evidence appears equally or less. -// - 0.0 when strict was ignored while tentative was preferred or no strict evidence exists. -// @intent measure whether the final natural-language answer favored strict matches over tentative ones. -func computeLLMStrictBias(result RunResult, query Query) float64 { - strictHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedStrictSymbols}) - tentativeHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedTentativeSymbols}) - if len(query.ExpectedStrictSymbols) == 0 { - return 1.0 - } - if strictHit == 0.0 { - return 0.0 - } - if len(query.ExpectedTentativeSymbols) == 0 || strictHit >= tentativeHit { - return 1.0 - } - if strictHit > 0.0 { - return 0.5 - } - return 0.0 -} - -// @intent quantify how much tentative-only evidence leaked into answers when strict expectations were missed. -func computeStrictContamination(result RunResult, query Query) float64 { - strictHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedStrictSymbols}) - tentativeHit := MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedTentativeSymbols}) - if len(query.ExpectedStrictSymbols) == 0 || len(query.ExpectedTentativeSymbols) == 0 { - return 0.0 - } - if strictHit > 0.0 { - return 0.0 - } - return tentativeHit -} - -// ComputeMatch derives a MatchResult from a single RunResult and its Query. -// @intent consolidate per-query benchmark scoring into one reusable result structure. -func ComputeMatch(result RunResult, query Query) MatchResult { - coverage := matchSymbolsFromToolOutputs(result, query.ExpectedStrictSymbols, query.ExpectedTentativeSymbols) - return MatchResult{ - QueryID: result.QueryID, - FileHitRatio: MatchFiles(result, query), - SymbolHitRatio: MatchSymbols(result, query), - StrictSymbolHitRatio: MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedStrictSymbols}), - TentativeSymbolHitRatio: MatchSymbols(result, Query{ExpectedSymbols: query.ExpectedTentativeSymbols}), - LLMStrictBias: computeLLMStrictBias(result, query), - StrictContaminationRate: computeStrictContamination(result, query), - ToolAwareStrictRatio: safeRatio(coverage.StrictMatched, coverage.StrictCount), - ToolAwareTentativeRatio: safeRatio(coverage.TentativeMatched, coverage.TentativeCount), - TotalToolCalls: len(result.ToolCalls), - CcgToolCalls: CountCcgToolCalls(result), - TotalInputTokens: result.InputTokens, - } -} - -// AnalyzeRun computes MatchResult for every result in the run against the corpus. -// Results with no matching query are skipped. -// @intent turn a full benchmark run into per-query scored matches against the corpus definition. -func AnalyzeRun(run *BenchmarkRun, corpus *Corpus) []MatchResult { - queryMap := make(map[string]Query, len(corpus.Queries)) - for _, q := range corpus.Queries { - queryMap[q.ID] = q - } - var matches []MatchResult - for _, r := range run.Results { - q, ok := queryMap[r.QueryID] - if !ok { - continue - } - matches = append(matches, ComputeMatch(r, q)) - } - return matches -} diff --git a/internal/benchmark/analyze_test.go b/internal/benchmark/analyze_test.go deleted file mode 100644 index 8b832a5..0000000 --- a/internal/benchmark/analyze_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package benchmark_test - -import ( - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestMatchFiles_AllMatch(t *testing.T) { - result := benchmark.RunResult{ - FilesRead: []string{"internal/webhook/handler.go", "internal/webhook/syncqueue.go"}, - } - query := benchmark.Query{ - ExpectedFiles: []string{"internal/webhook/handler.go", "internal/webhook/syncqueue.go"}, - } - ratio := benchmark.MatchFiles(result, query) - if ratio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0", ratio) - } -} - -func TestMatchFiles_PartialMatch(t *testing.T) { - result := benchmark.RunResult{ - FilesRead: []string{"internal/webhook/handler.go"}, - } - query := benchmark.Query{ - ExpectedFiles: []string{"internal/webhook/handler.go", "internal/webhook/syncqueue.go", "internal/webhook/auth.go"}, - } - ratio := benchmark.MatchFiles(result, query) - want := 1.0 / 3.0 - if ratio < want-0.001 || ratio > want+0.001 { - t.Errorf("FileHitRatio: got %.3f, want %.3f", ratio, want) - } -} - -func TestMatchFiles_NoExpected(t *testing.T) { - result := benchmark.RunResult{FilesRead: []string{"any.go"}} - query := benchmark.Query{ExpectedFiles: nil} - ratio := benchmark.MatchFiles(result, query) - if ratio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0 when no expected files", ratio) - } -} - -func TestMatchFiles_AbsolutePathSuffix(t *testing.T) { - result := benchmark.RunResult{ - FilesRead: []string{"/Users/user/repo/internal/webhook/handler.go"}, - } - query := benchmark.Query{ - ExpectedFiles: []string{"internal/webhook/handler.go"}, - } - ratio := benchmark.MatchFiles(result, query) - if ratio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0 for absolute path suffix match", ratio) - } -} - -func TestMatchSymbols_InAnswer(t *testing.T) { - result := benchmark.RunResult{Answer: "WebhookHandler handles push events via SyncQueue"} - query := benchmark.Query{ExpectedSymbols: []string{"WebhookHandler", "SyncQueue"}} - ratio := benchmark.MatchSymbols(result, query) - if ratio != 1.0 { - t.Errorf("SymbolHitRatio: got %.3f, want 1.0", ratio) - } -} - -func TestMatchSymbols_PartialMatch(t *testing.T) { - result := benchmark.RunResult{Answer: "WebhookHandler handles events"} - query := benchmark.Query{ExpectedSymbols: []string{"WebhookHandler", "SyncQueue"}} - ratio := benchmark.MatchSymbols(result, query) - if ratio != 0.5 { - t.Errorf("SymbolHitRatio: got %.3f, want 0.5", ratio) - } -} - -func TestMatchSymbolsFromToolOutputs_StrictTentative(t *testing.T) { - toolOut := `{"pattern":"callers_of","results":[{"qualified_name":"StrictA","confidence":"strict"},{"qualified_name":"TentativeA","confidence":"tentative"},{"qualified_name":"TentativeA","confidence":"tentative"}]}` - result := benchmark.RunResult{ - QueryID: "q1", - Answer: "StrictA", - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__query_graph", Output: toolOut}, - }, - } - query := benchmark.Query{ - ExpectedStrictSymbols: []string{"StrictA"}, - ExpectedTentativeSymbols: []string{"TentativeA", "TentativeMissing"}, - } - m := benchmark.ComputeMatch(result, query) - if m.StrictSymbolHitRatio != 1.0 { - t.Errorf("StrictSymbolHitRatio: got %.3f, want 1.0", m.StrictSymbolHitRatio) - } - if m.TentativeSymbolHitRatio != 0.0 { - t.Errorf("TentativeSymbolHitRatio: got %.3f, want 0.0", m.TentativeSymbolHitRatio) - } - if m.ToolAwareStrictRatio != 1.0 { - t.Errorf("ToolAwareStrictRatio: got %.3f, want 1.0", m.ToolAwareStrictRatio) - } - if m.ToolAwareTentativeRatio != 0.5 { - t.Errorf("ToolAwareTentativeRatio: got %.3f, want 0.5", m.ToolAwareTentativeRatio) - } -} - -func TestLLMStrictBias_Contamination(t *testing.T) { - result := benchmark.RunResult{ - Answer: "TentativeA 사용 지점만 확인", - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__query_graph", Output: `{"pattern":"callees_of","results":[]}`}, - }, - } - query := benchmark.Query{ - ExpectedStrictSymbols: []string{"StrictA"}, - ExpectedTentativeSymbols: []string{"TentativeA"}, - } - m := benchmark.ComputeMatch(result, query) - if m.LLMStrictBias != 0.0 { - t.Errorf("LLMStrictBias: got %.3f, want 0.0", m.LLMStrictBias) - } - if m.StrictContaminationRate != 1.0 { - t.Errorf("StrictContaminationRate: got %.3f, want 1.0", m.StrictContaminationRate) - } -} - -func TestCountCcgToolCalls(t *testing.T) { - result := benchmark.RunResult{ - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__search"}, - {Tool: "mcp__ccg__get_node"}, - {Tool: "Read"}, - {Tool: "Grep"}, - }, - } - count := benchmark.CountCcgToolCalls(result) - if count != 2 { - t.Errorf("CcgToolCalls: got %d, want 2", count) - } -} - -func TestComputeMatch_FullResult(t *testing.T) { - toolOut := `{"pattern":"callees_of","results":[{"qualified_name":"WebhookHandler","confidence":"strict"},{"qualified_name":"SyncQueue","confidence":"tentative"}]}` - result := benchmark.RunResult{ - QueryID: "q1", - FilesRead: []string{"internal/webhook/handler.go"}, - Answer: "WebhookHandler 구현이 " + strings.Repeat("x", 10), - InputTokens: 200, - ToolCalls: []benchmark.ToolCall{ - {Tool: "mcp__ccg__search"}, - {Tool: "mcp__ccg__query_graph", Output: toolOut}, - {Tool: "Read"}, - }, - } - query := benchmark.Query{ - ID: "q1", - ExpectedFiles: []string{"internal/webhook/handler.go"}, - ExpectedSymbols: []string{"WebhookHandler"}, - ExpectedStrictSymbols: []string{"WebhookHandler"}, - ExpectedTentativeSymbols: []string{"SyncQueue"}, - } - m := benchmark.ComputeMatch(result, query) - if m.QueryID != "q1" { - t.Errorf("QueryID: got %q", m.QueryID) - } - if m.FileHitRatio != 1.0 { - t.Errorf("FileHitRatio: got %.3f, want 1.0", m.FileHitRatio) - } - if m.SymbolHitRatio != 1.0 { - t.Errorf("SymbolHitRatio: got %.3f, want 1.0", m.SymbolHitRatio) - } - if m.StrictSymbolHitRatio != 1.0 { - t.Errorf("StrictSymbolHitRatio: got %.3f, want 1.0", m.StrictSymbolHitRatio) - } - if m.TentativeSymbolHitRatio != 0.0 { - t.Errorf("TentativeSymbolHitRatio: got %.3f, want 0.0", m.TentativeSymbolHitRatio) - } - if m.ToolAwareStrictRatio != 1.0 { - t.Errorf("ToolAwareStrictRatio: got %.3f, want 1.0", m.ToolAwareStrictRatio) - } - if m.ToolAwareTentativeRatio != 1.0 { - t.Errorf("ToolAwareTentativeRatio: got %.3f, want 1.0", m.ToolAwareTentativeRatio) - } - if m.LLMStrictBias != 1.0 { - t.Errorf("LLMStrictBias: got %.3f, want 1.0", m.LLMStrictBias) - } - if m.StrictContaminationRate != 0.0 { - t.Errorf("StrictContaminationRate: got %.3f, want 0.0", m.StrictContaminationRate) - } - if m.TotalToolCalls != 3 { - t.Errorf("TotalToolCalls: got %d, want 3", m.TotalToolCalls) - } - if m.CcgToolCalls != 2 { - t.Errorf("CcgToolCalls: got %d, want 2", m.CcgToolCalls) - } - if m.TotalInputTokens != 200 { - t.Errorf("TotalInputTokens: got %d, want 200", m.TotalInputTokens) - } -} - -func TestAnalyzeRun_MultipleResults(t *testing.T) { - run := &benchmark.BenchmarkRun{ - Mode: "with-ccg", - Results: []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}, Answer: "Foo"}, - {QueryID: "q2", FilesRead: []string{"b.go"}, Answer: "Bar"}, - }, - } - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go"}, ExpectedSymbols: []string{"Foo"}}, - {ID: "q2", ExpectedFiles: []string{"b.go"}, ExpectedSymbols: []string{"Bar"}}, - }, - } - matches := benchmark.AnalyzeRun(run, corpus) - if len(matches) != 2 { - t.Errorf("expected 2 matches, got %d", len(matches)) - } -} diff --git a/internal/benchmark/compare.go b/internal/benchmark/compare.go deleted file mode 100644 index 931e05b..0000000 --- a/internal/benchmark/compare.go +++ /dev/null @@ -1,17 +0,0 @@ -// @index Comparison helpers for benchmark runs with and without CCG. -package benchmark - -// Compare builds a ComparisonReport from a with-ccg run and an optional without-ccg run. -// Matches contains with-ccg metrics; MatchesWithout contains without-ccg metrics when provided. -// @intent produce one report that captures both benchmark modes and their scored query matches. -func Compare(withCCG *BenchmarkRun, withoutCCG *BenchmarkRun, corpus *Corpus) *ComparisonReport { - report := &ComparisonReport{ - WithCCG: withCCG, - WithoutCCG: withoutCCG, - Matches: AnalyzeRun(withCCG, corpus), - } - if withoutCCG != nil { - report.MatchesWithout = AnalyzeRun(withoutCCG, corpus) - } - return report -} diff --git a/internal/benchmark/compare_test.go b/internal/benchmark/compare_test.go deleted file mode 100644 index b2a6491..0000000 --- a/internal/benchmark/compare_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package benchmark_test - -import ( - "testing" - "time" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func makeRun(mode string, results []benchmark.RunResult) *benchmark.BenchmarkRun { - return &benchmark.BenchmarkRun{Mode: mode, RunAt: time.Now(), Results: results} -} - -func TestCompare_BothRuns(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go"}, ExpectedSymbols: []string{"Foo"}}, - }, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}, Answer: "Foo is here", InputTokens: 100}, - }) - withoutCCG := makeRun("without-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{}, Answer: "not sure", InputTokens: 200}, - }) - report := benchmark.Compare(withCCG, withoutCCG, corpus) - if report.WithCCG == nil { - t.Error("WithCCG should not be nil") - } - if report.WithoutCCG == nil { - t.Error("WithoutCCG should not be nil") - } - if len(report.Matches) == 0 { - t.Error("expected at least one match result") - } -} - -func TestCompare_OnlyWithCCG(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "test"}, - }, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", Answer: "answer"}, - }) - report := benchmark.Compare(withCCG, nil, corpus) - if report.WithCCG == nil { - t.Error("WithCCG should not be nil") - } - if report.WithoutCCG != nil { - t.Errorf("WithoutCCG should be nil, got %+v", report.WithoutCCG) - } -} - -func TestCompare_TokenDiff(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{{ID: "q1"}}, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", InputTokens: 50}, - }) - withoutCCG := makeRun("without-ccg", []benchmark.RunResult{ - {QueryID: "q1", InputTokens: 300}, - }) - report := benchmark.Compare(withCCG, withoutCCG, corpus) - if len(report.Matches) == 0 { - t.Fatal("no matches") - } - if report.Matches[0].TotalInputTokens != 50 { - t.Errorf("Matches should reflect with-ccg tokens (50), got %d", report.Matches[0].TotalInputTokens) - } -} - -func TestCompare_FileHitImprovement(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go", "b.go"}}, - }, - } - withCCG := makeRun("with-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go", "b.go"}}, - }) - withoutCCG := makeRun("without-ccg", []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}}, - }) - report := benchmark.Compare(withCCG, withoutCCG, corpus) - if len(report.Matches) == 0 { - t.Fatal("no matches") - } - if report.Matches[0].FileHitRatio != 1.0 { - t.Errorf("with-ccg FileHitRatio should be 1.0, got %.3f", report.Matches[0].FileHitRatio) - } -} diff --git a/internal/benchmark/corpus.go b/internal/benchmark/corpus.go deleted file mode 100644 index f113a23..0000000 --- a/internal/benchmark/corpus.go +++ /dev/null @@ -1,71 +0,0 @@ -// @index Benchmark corpus loading, saving, and validation utilities. -package benchmark - -import ( - "fmt" - "os" - "regexp" - "strings" - - "go.yaml.in/yaml/v3" -) - -var validQueryID = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) - -// LoadCorpus reads a queries.yaml file and validates its contents. -// @intent load a benchmark corpus from disk while enforcing schema and ID constraints up front. -func LoadCorpus(path string) (*Corpus, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("read corpus %q: %w", path, err) - } - var c Corpus - if err := yaml.Unmarshal(data, &c); err != nil { - return nil, fmt.Errorf("parse corpus %q: %w", path, err) - } - if err := ValidateCorpus(&c); err != nil { - return nil, err - } - return &c, nil -} - -// SaveCorpus writes a Corpus to a YAML file. -// @intent persist benchmark corpus definitions in the YAML format used by the CLI. -func SaveCorpus(path string, c *Corpus) error { - data, err := yaml.Marshal(c) - if err != nil { - return fmt.Errorf("marshal corpus: %w", err) - } - if err := os.WriteFile(path, data, 0o644); err != nil { - return fmt.Errorf("write corpus %q: %w", path, err) - } - return nil -} - -// ValidateCorpus checks that all queries have required fields and no duplicate IDs. -// @intent reject malformed benchmark corpora before any benchmark execution depends on them. -func ValidateCorpus(c *Corpus) error { - if len(c.Queries) == 0 { - return fmt.Errorf("corpus has no queries") - } - seen := make(map[string]bool, len(c.Queries)) - for i, q := range c.Queries { - if q.ID == "" { - return fmt.Errorf("query[%d] missing id", i) - } - if !validQueryID.MatchString(q.ID) { - return fmt.Errorf("query[%d] id %q contains invalid characters (use only [A-Za-z0-9_.-])", i, q.ID) - } - if q.Description == "" { - return fmt.Errorf("query %q missing description", q.ID) - } - if seen[q.ID] { - return fmt.Errorf("duplicate query id %q", q.ID) - } - seen[q.ID] = true - if strings.Contains(q.Description, markerStart) || strings.Contains(q.Description, markerEnd) { - return fmt.Errorf("query %q description contains reserved benchmark marker", q.ID) - } - } - return nil -} diff --git a/internal/benchmark/corpus_test.go b/internal/benchmark/corpus_test.go deleted file mode 100644 index 4c30708..0000000 --- a/internal/benchmark/corpus_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package benchmark_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestLoadCorpus_ValidYAML(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `version: "1" -queries: - - id: q1 - description: "결제 실패 복구 로직" - expected_files: - - internal/webhook/handler.go - expected_symbols: - - WebhookHandler - difficulty: medium -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - corpus, err := benchmark.LoadCorpus(path) - if err != nil { - t.Fatalf("LoadCorpus: %v", err) - } - if len(corpus.Queries) != 1 { - t.Errorf("Queries count: got %d, want 1", len(corpus.Queries)) - } - if corpus.Queries[0].ID != "q1" { - t.Errorf("ID: got %q, want q1", corpus.Queries[0].ID) - } -} - -func TestLoadCorpus_FileNotFound(t *testing.T) { - _, err := benchmark.LoadCorpus("/nonexistent/queries.yaml") - if err == nil { - t.Error("expected error for missing file") - } -} - -func TestLoadCorpus_MissingID(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - description: "no id here" -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - _, err := benchmark.LoadCorpus(path) - if err == nil { - t.Error("expected validation error for missing id") - } -} - -func TestLoadCorpus_MissingDescription(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - _, err := benchmark.LoadCorpus(path) - if err == nil { - t.Error("expected validation error for missing description") - } -} - -func TestLoadCorpus_DuplicateID(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 - description: "first" - - id: q1 - description: "duplicate" -` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - _, err := benchmark.LoadCorpus(path) - if err == nil { - t.Error("expected validation error for duplicate id") - } -} - -func TestSaveCorpus_WritesYAML(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - corpus := &benchmark.Corpus{ - Version: "1", - Queries: []benchmark.Query{ - {ID: "q1", Description: "테스트 쿼리", Difficulty: "easy"}, - }, - } - if err := benchmark.SaveCorpus(path, corpus); err != nil { - t.Fatalf("SaveCorpus: %v", err) - } - loaded, err := benchmark.LoadCorpus(path) - if err != nil { - t.Fatalf("LoadCorpus after save: %v", err) - } - if len(loaded.Queries) != 1 || loaded.Queries[0].ID != "q1" { - t.Errorf("round-trip failed: got %+v", loaded) - } -} - -func TestValidateCorpus_ValidQuery(t *testing.T) { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "valid query"}, - }, - } - if err := benchmark.ValidateCorpus(corpus); err != nil { - t.Errorf("expected nil error for valid corpus, got: %v", err) - } -} - -func TestValidateCorpus_EmptyCorpus(t *testing.T) { - corpus := &benchmark.Corpus{Queries: []benchmark.Query{}} - if err := benchmark.ValidateCorpus(corpus); err == nil { - t.Error("expected error for empty corpus") - } -} diff --git a/internal/benchmark/jsonl.go b/internal/benchmark/jsonl.go deleted file mode 100644 index c08455f..0000000 --- a/internal/benchmark/jsonl.go +++ /dev/null @@ -1,284 +0,0 @@ -// @index JSONL session parsing and extraction helpers for benchmark analysis. -package benchmark - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "regexp" - "strings" -) - -var ( - reMarkerStart = regexp.MustCompile(`^===BENCHMARK_QUERY_START id=([A-Za-z0-9_.-]+)===$`) - reMarkerEnd = regexp.MustCompile(`^===BENCHMARK_QUERY_END id=([A-Za-z0-9_.-]+)===$`) -) - -// SessionMessage represents one line from a Claude Code session JSONL file. -// @intent model the subset of Claude session JSONL needed to reconstruct benchmark runs. -type SessionMessage struct { - Type string `json:"type"` - Message *MessagePayload `json:"message,omitempty"` - Content json.RawMessage `json:"content,omitempty"` - ToolUseID string `json:"tool_use_id,omitempty"` - IsError bool `json:"is_error,omitempty"` -} - -// MessagePayload is the nested message object inside a SessionMessage. -// @intent expose assistant role, content blocks, and usage data from session lines. -type MessagePayload struct { - Role string `json:"role"` - Content []ContentBlock `json:"content"` - Usage *UsageInfo `json:"usage,omitempty"` -} - -// ContentBlock is a single content item (text, tool_use, etc.). -// @intent represent individual Claude message blocks so benchmark analyzers can inspect tool calls and text. -type ContentBlock struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - Name string `json:"name,omitempty"` // tool_use - ID string `json:"id,omitempty"` - Input json.RawMessage `json:"input,omitempty"` -} - -// UsageInfo holds token usage from Claude's response. -// @intent capture token accounting for one Claude message so benchmark runs can aggregate usage. -type UsageInfo struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -// QuerySegment groups the messages belonging to a single benchmark query. -// @intent isolate the portion of a session that belongs to one marked benchmark query. -type QuerySegment struct { - QueryID string - Messages []SessionMessage -} - -// ParseJSONL reads a Claude Code session JSONL file and returns all parsed lines. -// Lines that are not valid JSON are silently skipped. -// @intent ingest recorded Claude sessions even when they contain partial or non-JSON noise lines. -func ParseJSONL(path string) ([]SessionMessage, error) { - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("open %q: %w", path, err) - } - defer f.Close() - - var msgs []SessionMessage - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 10<<20), 10<<20) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - var m SessionMessage - if err := json.Unmarshal([]byte(line), &m); err != nil { - continue // skip invalid lines - } - msgs = append(msgs, m) - } - return msgs, scanner.Err() -} - -const ( - markerStart = "===BENCHMARK_QUERY_START id=" - markerEnd = "===BENCHMARK_QUERY_END id=" -) - -// extractMarker checks if a SessionMessage contains a benchmark marker. -// Returns (queryID, isStart, found). Uses strict regex to prevent injection via crafted IDs. -// @intent locate benchmark query boundaries inside session transcripts safely. -func extractMarker(m SessionMessage) (queryID string, isStart bool, found bool) { - var text string - if m.Type == "tool_result" { - text = parseFirstText(m.Content) - } else if m.Message != nil { - for _, b := range m.Message.Content { - if b.Type == "text" { - text = b.Text - break - } - } - } - text = strings.TrimSpace(text) - if sub := reMarkerStart.FindStringSubmatch(text); sub != nil { - return sub[1], true, true - } - if sub := reMarkerEnd.FindStringSubmatch(text); sub != nil { - return sub[1], false, true - } - return "", false, false -} - -// ExtractQuerySegments splits a session's messages into per-query segments using markers. -// An unclosed segment (START without END) is included, spanning to the last message. -// @intent reconstruct per-query transcript slices from a whole benchmark session log. -func ExtractQuerySegments(msgs []SessionMessage) ([]QuerySegment, error) { - var segs []QuerySegment - var current *QuerySegment - - for _, m := range msgs { - id, isStart, found := extractMarker(m) - if found { - if isStart { - current = &QuerySegment{QueryID: id} - } else if current != nil && current.QueryID == id { - segs = append(segs, *current) - current = nil - } - continue - } - if current != nil { - current.Messages = append(current.Messages, m) - } - } - // Include unclosed segment - if current != nil { - segs = append(segs, *current) - } - return segs, nil -} - -// ExtractToolCalls collects all tool_use blocks from a segment's messages. -// @intent recover structured tool invocation history for one benchmark query. -func ExtractToolCalls(seg QuerySegment) []ToolCall { - outputs := extractToolOutputs(seg.Messages) - var calls []ToolCall - for _, m := range seg.Messages { - if m.Message == nil { - continue - } - for _, b := range m.Message.Content { - if b.Type == "tool_use" { - var inputStr string - if len(b.Input) > 0 { - inputStr = string(b.Input) - } - call := ToolCall{ - Tool: b.Name, - Input: inputStr, - } - if b.ID != "" { - call.ToolUseID = b.ID - call.Output = outputs[b.ID] - } - calls = append(calls, call) - } - } - } - return calls -} - -// ExtractFilesRead returns deduplicated file paths from Read tool calls in the segment. -// @intent estimate file-inspection behavior from tool-use records without double-counting reads. -func ExtractFilesRead(seg QuerySegment) []string { - var files []string - seen := make(map[string]bool) - for _, m := range seg.Messages { - if m.Message == nil { - continue - } - for _, b := range m.Message.Content { - if b.Type != "tool_use" || b.Name != "Read" { - continue - } - var inp struct { - FilePath string `json:"file_path"` - } - if err := json.Unmarshal(b.Input, &inp); err == nil && inp.FilePath != "" && !seen[inp.FilePath] { - seen[inp.FilePath] = true - files = append(files, inp.FilePath) - } - } - } - return files -} - -// @intent map tool_use ids to their first text payload so later benchmark analysis can inspect tool outputs by invocation. -func extractToolOutputs(messages []SessionMessage) map[string]string { - outputs := make(map[string]string) - for _, m := range messages { - if m.Type != "tool_result" || m.ToolUseID == "" { - continue - } - outputs[m.ToolUseID] = parseFirstText(m.Content) - } - return outputs -} - -// @intent normalize heterogeneous tool result payloads into one comparable trimmed text string. -func parseFirstText(raw json.RawMessage) string { - if len(raw) == 0 { - return "" -} - - var text string - if err := json.Unmarshal(raw, &text); err == nil { - return strings.TrimSpace(text) - } - - var blocks []ContentBlock - if err := json.Unmarshal(raw, &blocks); err == nil { - for _, b := range blocks { - if b.Type == "text" && b.Text != "" { - return strings.TrimSpace(b.Text) - } - } - return "" - } - - var block ContentBlock - if err := json.Unmarshal(raw, &block); err == nil { - if block.Type == "text" { - return strings.TrimSpace(block.Text) - } - } - return "" -} - -// ExtractTokens sums input and output token counts across all messages in a segment. -// @intent aggregate token usage for the full lifecycle of one benchmark query. -func ExtractTokens(seg QuerySegment) (inputTokens, outputTokens int) { - for _, m := range seg.Messages { - if m.Message != nil && m.Message.Usage != nil { - inputTokens += m.Message.Usage.InputTokens - outputTokens += m.Message.Usage.OutputTokens - } - } - return -} - -// ExtractAnswer returns the text of the last assistant text block in the segment. -// @intent capture the final assistant answer that should be evaluated for symbol hits. -func ExtractAnswer(seg QuerySegment) string { - var last string - for _, m := range seg.Messages { - if m.Message == nil || m.Message.Role != "assistant" { - continue - } - for _, b := range m.Message.Content { - if b.Type == "text" && b.Text != "" { - last = b.Text - } - } - } - return last -} - -// ExtractRunResult builds a RunResult from a query segment. -// @intent convert an extracted query segment into the canonical benchmark run result shape. -func ExtractRunResult(queryID string, seg QuerySegment) RunResult { - in, out := ExtractTokens(seg) - return RunResult{ - QueryID: queryID, - ToolCalls: ExtractToolCalls(seg), - FilesRead: ExtractFilesRead(seg), - Answer: ExtractAnswer(seg), - InputTokens: in, - OutputTokens: out, - } -} diff --git a/internal/benchmark/jsonl_test.go b/internal/benchmark/jsonl_test.go deleted file mode 100644 index 7d09ea1..0000000 --- a/internal/benchmark/jsonl_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package benchmark_test - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -// sampleJSONL returns lines representing a minimal Claude Code session JSONL. -// It includes START/END markers inside tool_result content. -func writeSampleJSONL(t *testing.T, lines []string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "session.jsonl") - var content string - for _, l := range lines { - content += l + "\n" - } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - return path -} - -func TestParseJSONL_ValidSession(t *testing.T) { - path := writeSampleJSONL(t, []string{ - `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"world"}],"usage":{"input_tokens":10,"output_tokens":5}}}`, - }) - msgs, err := benchmark.ParseJSONL(path) - if err != nil { - t.Fatalf("ParseJSONL: %v", err) - } - if len(msgs) != 2 { - t.Errorf("got %d messages, want 2", len(msgs)) - } -} - -func TestParseJSONL_EmptyFile(t *testing.T) { - path := writeSampleJSONL(t, nil) - msgs, err := benchmark.ParseJSONL(path) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(msgs) != 0 { - t.Errorf("expected 0 messages, got %d", len(msgs)) - } -} - -func TestParseJSONL_InvalidLine(t *testing.T) { - path := writeSampleJSONL(t, []string{ - `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]}}`, - `NOT VALID JSON {{{`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"ok"}]}}`, - }) - msgs, err := benchmark.ParseJSONL(path) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(msgs) != 2 { - t.Errorf("expected 2 valid messages (invalid skipped), got %d", len(msgs)) - } -} - -// markerLine produces a tool_result line containing a benchmark marker. -func markerLine(markerText string) string { - return `{"type":"tool_result","tool_use_id":"x","content":"` + markerText + `"}` -} - -func TestExtractQuerySegments_TwoMarkers(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"answer1"}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - markerLine(`===BENCHMARK_QUERY_START id=q2===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"answer2"}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q2===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - t.Fatalf("ExtractQuerySegments: %v", err) - } - if len(segs) != 2 { - t.Errorf("expected 2 segments, got %d", len(segs)) - } - if segs[0].QueryID != "q1" { - t.Errorf("seg[0].QueryID: got %q, want q1", segs[0].QueryID) - } - if segs[1].QueryID != "q2" { - t.Errorf("seg[1].QueryID: got %q, want q2", segs[1].QueryID) - } -} - -func TestExtractQuerySegments_NoMarkers(t *testing.T) { - path := writeSampleJSONL(t, []string{ - `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`, - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - t.Fatalf("ExtractQuerySegments: %v", err) - } - if len(segs) != 0 { - t.Errorf("expected 0 segments, got %d", len(segs)) - } -} - -func TestExtractQuerySegments_UnclosedMarker(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"partial answer"}]}}`, - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - t.Fatalf("ExtractQuerySegments: %v", err) - } - if len(segs) != 1 { - t.Errorf("expected 1 open segment, got %d", len(segs)) - } - if segs[0].QueryID != "q1" { - t.Errorf("seg.QueryID: got %q, want q1", segs[0].QueryID) - } -} - -func TestExtractToolCalls_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"mcp__ccg__search","input":{"query":"retry"}}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t2","name":"Read","input":{"file_path":"internal/foo.go"}}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - if len(segs) == 0 { - t.Fatal("no segments") - } - calls := benchmark.ExtractToolCalls(segs[0]) - if len(calls) != 2 { - t.Errorf("expected 2 tool calls, got %d", len(calls)) - } - if calls[0].Tool != "mcp__ccg__search" { - t.Errorf("calls[0].Tool: got %q", calls[0].Tool) - } -} - -func TestExtractToolCalls_WithToolResultOutput(t *testing.T) { - t.Helper() - toolResult := `{"pattern":"callers_of","results":[{"qualified_name":"Foo","confidence":"strict"}]}` - toolResultJSON, err := json.Marshal(toolResult) - if err != nil { - t.Fatalf("json.Marshal: %v", err) - } - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"x1","name":"mcp__ccg__query_graph","input":{"pattern":"callers_of","target":"pkg.Foo"}}]}}`, - `{"type":"tool_result","tool_use_id":"x1","content":` + string(toolResultJSON) + `}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - calls := benchmark.ExtractToolCalls(segs[0]) - if len(calls) != 1 { - t.Fatalf("expected 1 tool call, got %d", len(calls)) - } - if calls[0].ToolUseID != "x1" { - t.Errorf("ToolUseID: got %q", calls[0].ToolUseID) - } - if calls[0].Output != toolResult { - t.Errorf("Output: got %q want %q", calls[0].Output, toolResult) - } -} - -func TestExtractToolCalls_WithToolResultArrayContent(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"x1","name":"mcp__ccg__query_graph","input":{"pattern":"callees_of","target":"pkg.Bar"}}]}}`, - `{"type":"tool_result","tool_use_id":"x1","content":[{"type":"text","text":"{\"pattern\":\"callees_of\",\"results\":[{\"qualified_name\":\"Bar\",\"confidence\":\"strict\"}]}"}]}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - calls := benchmark.ExtractToolCalls(segs[0]) - if len(calls) != 1 { - t.Fatalf("expected 1 tool call, got %d", len(calls)) - } - if calls[0].Output == "" { - t.Fatalf("tool output should be captured") - } - if !strings.Contains(calls[0].Output, `"qualified_name":"Bar"`) { - t.Errorf("tool output not captured as JSON text: %q", calls[0].Output) - } -} - -func TestExtractFilesRead_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"Read","input":{"file_path":"internal/webhook/handler.go"}}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t2","name":"Glob","input":{"pattern":"internal/**/*.go"}}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - files := benchmark.ExtractFilesRead(segs[0]) - if len(files) != 1 { - t.Errorf("expected 1 file (only Read, not Glob), got %d: %v", len(files), files) - } - if files[0] != "internal/webhook/handler.go" { - t.Errorf("files[0]: got %q", files[0]) - } -} - -func TestExtractTokens_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"first"}],"usage":{"input_tokens":100,"output_tokens":20}}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"second"}],"usage":{"input_tokens":50,"output_tokens":10}}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - in, out := benchmark.ExtractTokens(segs[0]) - if in != 150 { - t.Errorf("InputTokens: got %d, want 150", in) - } - if out != 30 { - t.Errorf("OutputTokens: got %d, want 30", out) - } -} - -func TestExtractAnswer_FromSegment(t *testing.T) { - path := writeSampleJSONL(t, []string{ - markerLine(`===BENCHMARK_QUERY_START id=q1===`), - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"first message"}]}}`, - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"final answer"}]}}`, - markerLine(`===BENCHMARK_QUERY_END id=q1===`), - }) - msgs, _ := benchmark.ParseJSONL(path) - segs, _ := benchmark.ExtractQuerySegments(msgs) - answer := benchmark.ExtractAnswer(segs[0]) - if answer != "final answer" { - t.Errorf("answer: got %q, want %q", answer, "final answer") - } -} diff --git a/internal/benchmark/report.go b/internal/benchmark/report.go deleted file mode 100644 index 2540da2..0000000 --- a/internal/benchmark/report.go +++ /dev/null @@ -1,53 +0,0 @@ -// @index Markdown report generation for benchmark comparisons. -package benchmark - -import ( - "fmt" - "os" - "strings" -) - -// WriteReport renders a ComparisonReport as markdown and writes it to outPath. -// @intent turn benchmark comparison data into a readable artifact for sharing and regression tracking. -func WriteReport(report *ComparisonReport, outPath string) error { - if report.WithCCG == nil { - return fmt.Errorf("comparison report missing with-ccg run") - } - var sb strings.Builder - - sb.WriteString("# CCG Benchmark Report\n\n") - - // Summary section - sb.WriteString("## Summary\n\n") - fmt.Fprintf(&sb, "- Mode: `%s`\n", report.WithCCG.Mode) - fmt.Fprintf(&sb, "- Queries: %d\n", len(report.WithCCG.Results)) - if report.WithoutCCG != nil { - fmt.Fprintf(&sb, "- Comparison mode: `%s` vs `%s`\n", report.WithCCG.Mode, report.WithoutCCG.Mode) - } - sb.WriteString("\n") - - // Per-query results table - sb.WriteString("## Query Results\n\n") - sb.WriteString("| Query ID | File Hit | Symbol Hit | Tool Calls | CCG Calls | Input Tokens |\n") - sb.WriteString("|----------|----------|------------|------------|-----------|-------------|\n") - for _, m := range report.Matches { - fmt.Fprintf(&sb, "| %s | %.2f | %.2f | %d | %d | %d |\n", - m.QueryID, m.FileHitRatio, m.SymbolHitRatio, - m.TotalToolCalls, m.CcgToolCalls, m.TotalInputTokens) - } - sb.WriteString("\n") - - if len(report.MatchesWithout) > 0 { - sb.WriteString("## Without CCG Results\n\n") - sb.WriteString("| Query ID | File Hit | Symbol Hit | Tool Calls | CCG Calls | Input Tokens |\n") - sb.WriteString("|----------|----------|------------|------------|-----------|-------------|\n") - for _, m := range report.MatchesWithout { - fmt.Fprintf(&sb, "| %s | %.2f | %.2f | %d | %d | %d |\n", - m.QueryID, m.FileHitRatio, m.SymbolHitRatio, - m.TotalToolCalls, m.CcgToolCalls, m.TotalInputTokens) - } - sb.WriteString("\n") - } - - return os.WriteFile(outPath, []byte(sb.String()), 0o644) -} diff --git a/internal/benchmark/report_test.go b/internal/benchmark/report_test.go deleted file mode 100644 index 56043c1..0000000 --- a/internal/benchmark/report_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package benchmark_test - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func makeReport(withoutCCG *benchmark.BenchmarkRun) *benchmark.ComparisonReport { - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", ExpectedFiles: []string{"a.go"}, ExpectedSymbols: []string{"Foo"}}, - {ID: "q2", ExpectedFiles: []string{"b.go"}, ExpectedSymbols: []string{"Bar"}}, - }, - } - withCCG := &benchmark.BenchmarkRun{ - Mode: "with-ccg", - RunAt: time.Now(), - Results: []benchmark.RunResult{ - {QueryID: "q1", FilesRead: []string{"a.go"}, Answer: "Foo is here", InputTokens: 100, - ToolCalls: []benchmark.ToolCall{{Tool: "mcp__ccg__search"}}}, - {QueryID: "q2", FilesRead: []string{}, Answer: "unknown", InputTokens: 150}, - }, - } - return benchmark.Compare(withCCG, withoutCCG, corpus) -} - -func TestReport_ContainsSummary(t *testing.T) { - report := makeReport(nil) - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport: %v", err) - } - data, _ := os.ReadFile(out) - if !strings.Contains(string(data), "## Summary") { - t.Errorf("report missing '## Summary' section:\n%s", string(data)) - } -} - -func TestReport_ContainsQueryTable(t *testing.T) { - report := makeReport(nil) - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport: %v", err) - } - data, _ := os.ReadFile(out) - content := string(data) - if !strings.Contains(content, "q1") { - t.Error("report missing query ID q1") - } - if !strings.Contains(content, "q2") { - t.Error("report missing query ID q2") - } -} - -func TestReport_NoWithoutCCG(t *testing.T) { - report := makeReport(nil) - if report.WithoutCCG != nil { - t.Fatal("test setup error: WithoutCCG should be nil") - } - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport with single run: %v", err) - } - data, _ := os.ReadFile(out) - if len(data) == 0 { - t.Error("expected non-empty report") - } -} - -func TestReport_WritesToFile(t *testing.T) { - report := makeReport(nil) - dir := t.TempDir() - out := filepath.Join(dir, "report.md") - if err := benchmark.WriteReport(report, out); err != nil { - t.Fatalf("WriteReport: %v", err) - } - if _, err := os.Stat(out); os.IsNotExist(err) { - t.Error("report file was not created") - } -} diff --git a/internal/benchmark/runner.go b/internal/benchmark/runner.go deleted file mode 100644 index 44e8373..0000000 --- a/internal/benchmark/runner.go +++ /dev/null @@ -1,181 +0,0 @@ -// @index Benchmark runner orchestration for Claude CLI subprocess execution. -package benchmark - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strconv" - "time" -) - -// Executor abstracts the subprocess execution so tests can inject a mock. -// dir sets the working directory for the subprocess; empty means inherit from parent. -// @intent decouple benchmark orchestration from the concrete Claude CLI process implementation. -type Executor interface { - Execute(ctx context.Context, args []string, prompt, dir string) ([]byte, error) -} - -// RunnerConfig holds configuration for a benchmark run. -// @intent collect execution mode, working directory, and timeout settings for one benchmark run. -type RunnerConfig struct { - Mode string // "with-ccg" | "without-ccg" - CWD string // benchmark working directory - MaxToolCalls int - TimeoutSec int -} - -// DefaultRunnerConfig returns a RunnerConfig with sensible defaults. -// @intent provide conservative benchmark defaults that work for most local runs. -func DefaultRunnerConfig() RunnerConfig { - return RunnerConfig{ - MaxToolCalls: 50, - TimeoutSec: 120, - } -} - -// claudeOutput is the structure of `claude -p --output-format json` output. -// @intent decode the subset of Claude CLI JSON output needed for benchmark scoring. -type claudeOutput struct { - IsError bool `json:"is_error"` - Result string `json:"result"` - Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - } `json:"usage"` -} - -// buildArgs constructs CLI args for a specific run. -// mcpConfigFile is the path to an empty MCP config file for without-ccg mode. -// @intent centralize Claude CLI argument construction for both benchmark modes. -func buildArgs(cfg RunnerConfig, mcpConfigFile string) []string { - args := []string{"-p", "--output-format", "json"} - if cfg.MaxToolCalls > 0 { - args = append(args, "--max-turns", strconv.Itoa(cfg.MaxToolCalls)) - } - if cfg.Mode == "without-ccg" { - args = append(args, "--strict-mcp-config", "--mcp-config", mcpConfigFile) - } - return args -} - -// BuildClaudeArgs constructs the CLI argument slice for `claude -p`. -// Working directory is set on the subprocess directly (not via --cwd flag). -// In "without-ccg" mode, --strict-mcp-config disables all MCP servers. -// NOTE: For actual runs, use Runner.Run which creates a real temp file for --mcp-config. -// @intent expose deterministic argument construction for tests and tooling that inspect benchmark invocation behavior. -func BuildClaudeArgs(cfg RunnerConfig) []string { - if cfg.Mode == "without-ccg" { - return buildArgs(cfg, `{"mcpServers":{}}`) - } - return buildArgs(cfg, "") -} - -// BuildPrompt creates the prompt string that wraps a query with benchmark markers. -// The markers allow the JSONL analyzer to locate query boundaries. -// @intent surround benchmark queries with markers so later session analysis can recover exact query segments. -func BuildPrompt(q Query) string { - return fmt.Sprintf( - "===BENCHMARK_QUERY_START id=%s===\n%s\n===BENCHMARK_QUERY_END id=%s===", - q.ID, q.Description, q.ID, - ) -} - -// Runner executes benchmark queries sequentially using the provided Executor. -// @intent own the end-to-end lifecycle of benchmark query execution for one mode. -type Runner struct { - cfg RunnerConfig - exec Executor -} - -// NewRunner creates a Runner with the given config and executor. -// @intent assemble benchmark orchestration from execution config and a subprocess adapter. -func NewRunner(cfg RunnerConfig, exec Executor) *Runner { - return &Runner{cfg: cfg, exec: exec} -} - -// Run executes each query in the corpus and returns a BenchmarkRun. -// Executor errors are captured in RunResult.Error rather than aborting the run. -// @intent execute the full benchmark corpus while preserving per-query failures in the final report. -func (r *Runner) Run(ctx context.Context, corpus *Corpus) (*BenchmarkRun, error) { - run := &BenchmarkRun{ - Mode: r.cfg.Mode, - RunAt: time.Now(), - Results: make([]RunResult, 0, len(corpus.Queries)), - } - - mcpConfigFile, cleanup, err := r.prepareMCPConfig() - if err != nil { - return nil, err - } - defer cleanup() - - args := buildArgs(r.cfg, mcpConfigFile) - for _, q := range corpus.Queries { - start := time.Now() - prompt := BuildPrompt(q) - var ( - out []byte - execErr error - ) - func() { - qctx := ctx - if r.cfg.TimeoutSec > 0 { - var cancel context.CancelFunc - qctx, cancel = context.WithTimeout(ctx, time.Duration(r.cfg.TimeoutSec)*time.Second) - defer cancel() - } - out, execErr = r.exec.Execute(qctx, args, prompt, r.cfg.CWD) - }() - elapsed := time.Since(start).Milliseconds() - - res := RunResult{ - QueryID: q.ID, - ElapsedMs: elapsed, - } - if execErr != nil { - res.Error = execErr.Error() - } else { - r.parseClaudeOutput(out, &res) - } - run.Results = append(run.Results, res) - } - return run, nil -} - -// prepareMCPConfig creates a temp JSON file for without-ccg mode so that -// --mcp-config receives a file path rather than inline JSON (avoiding the -// variadic flag consuming the prompt from stdin). -// @intent disable MCP cleanly in without-ccg mode without confusing the Claude CLI argument parser. -func (r *Runner) prepareMCPConfig() (path string, cleanup func(), err error) { - if r.cfg.Mode != "without-ccg" { - return "", func() {}, nil - } - f, ferr := os.CreateTemp("", "ccg-bench-*.json") - if ferr != nil { - return "", nil, fmt.Errorf("create mcp config: %w", ferr) - } - if _, werr := f.WriteString(`{"mcpServers":{}}`); werr != nil { - f.Close() - os.Remove(f.Name()) - return "", nil, fmt.Errorf("write mcp config: %w", werr) - } - f.Close() - return f.Name(), func() { os.Remove(f.Name()) }, nil -} - -// parseClaudeOutput extracts token counts and answer text from claude's JSON output. -// @intent translate Claude CLI JSON output into benchmark result fields without failing the whole run on parse misses. -func (r *Runner) parseClaudeOutput(out []byte, res *RunResult) { - if len(out) == 0 { - return - } - var co claudeOutput - if err := json.Unmarshal(out, &co); err != nil { - return - } - res.InputTokens = co.Usage.InputTokens - res.OutputTokens = co.Usage.OutputTokens - res.Answer = co.Result -} diff --git a/internal/benchmark/runner_test.go b/internal/benchmark/runner_test.go deleted file mode 100644 index a4f93ef..0000000 --- a/internal/benchmark/runner_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package benchmark_test - -import ( - "context" - "errors" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestRunnerConfig_Defaults(t *testing.T) { - cfg := benchmark.DefaultRunnerConfig() - if cfg.MaxToolCalls != 50 { - t.Errorf("MaxToolCalls: got %d, want 50", cfg.MaxToolCalls) - } - if cfg.TimeoutSec != 120 { - t.Errorf("TimeoutSec: got %d, want 120", cfg.TimeoutSec) - } -} - -func TestBuildClaudeArgs_WithCWD(t *testing.T) { - // CWD is passed to Executor.Execute as dir, not as a --cwd CLI flag. - cfg := benchmark.RunnerConfig{ - Mode: "with-ccg", - CWD: "/tmp/benchmark-run", - MaxToolCalls: 50, - TimeoutSec: 120, - } - args := benchmark.BuildClaudeArgs(cfg) - for _, a := range args { - if a == "--cwd" { - t.Errorf("--cwd flag must not appear in args (set via cmd.Dir): %v", args) - } - } -} - -func TestBuildClaudeArgs_OutputFormat(t *testing.T) { - cfg := benchmark.DefaultRunnerConfig() - args := benchmark.BuildClaudeArgs(cfg) - found := false - for i, a := range args { - if a == "--output-format" && i+1 < len(args) && args[i+1] == "json" { - found = true - } - } - if !found { - t.Errorf("expected --output-format json in args: %v", args) - } -} - -// mockExecutor implements benchmark.Executor for testing. -type mockExecutor struct { - calls int - err error - output []byte -} - -func (m *mockExecutor) Execute(_ context.Context, _ []string, _, _ string) ([]byte, error) { - m.calls++ - if m.err != nil { - return nil, m.err - } - return m.output, nil -} - -func TestMockRunner_ExecutesEachQuery(t *testing.T) { - exec := &mockExecutor{output: []byte(`{}`)} - cfg := benchmark.DefaultRunnerConfig() - runner := benchmark.NewRunner(cfg, exec) - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "query one"}, - {ID: "q2", Description: "query two"}, - {ID: "q3", Description: "query three"}, - }, - } - run, err := runner.Run(context.Background(), corpus) - if err != nil { - t.Fatalf("Run: %v", err) - } - if len(run.Results) != 3 { - t.Errorf("expected 3 results, got %d", len(run.Results)) - } - if exec.calls != 3 { - t.Errorf("expected 3 executor calls, got %d", exec.calls) - } -} - -func TestMockRunner_HandlesError(t *testing.T) { - exec := &mockExecutor{err: errors.New("claude not found")} - cfg := benchmark.DefaultRunnerConfig() - runner := benchmark.NewRunner(cfg, exec) - corpus := &benchmark.Corpus{ - Queries: []benchmark.Query{ - {ID: "q1", Description: "fail query"}, - }, - } - run, err := runner.Run(context.Background(), corpus) - if err != nil { - t.Fatalf("Run should not return error (errors go into RunResult): %v", err) - } - if len(run.Results) != 1 { - t.Fatalf("expected 1 result, got %d", len(run.Results)) - } - if run.Results[0].Error == "" { - t.Error("expected RunResult.Error to be set on executor error") - } -} - -func TestBuildPromptWithMarkers(t *testing.T) { - q := benchmark.Query{ID: "q1", Description: "결제 실패 복구 로직"} - prompt := benchmark.BuildPrompt(q) - if !strings.Contains(prompt, "===BENCHMARK_QUERY_START id=q1===") { - t.Errorf("prompt missing START marker: %q", prompt) - } - if !strings.Contains(prompt, "===BENCHMARK_QUERY_END id=q1===") { - t.Errorf("prompt missing END marker: %q", prompt) - } - if !strings.Contains(prompt, q.Description) { - t.Errorf("prompt missing description: %q", prompt) - } -} diff --git a/internal/benchmark/schema.go b/internal/benchmark/schema.go deleted file mode 100644 index 0a4fd7e..0000000 --- a/internal/benchmark/schema.go +++ /dev/null @@ -1,90 +0,0 @@ -// Package benchmark provides types and utilities for benchmarking ccg MCP tool effectiveness. -package benchmark - -import "time" - -// Query represents a single benchmark query with expected results. -// @intent define one benchmark prompt and the files or symbols it is expected to surface. -type Query struct { - ID string `yaml:"id" json:"id"` - Description string `yaml:"description" json:"description"` - ExpectedFiles []string `yaml:"expected_files" json:"expected_files,omitempty"` - ExpectedSymbols []string `yaml:"expected_symbols" json:"expected_symbols,omitempty"` - ExpectedStrictSymbols []string `yaml:"expected_strict_symbols" json:"expected_strict_symbols,omitempty"` - ExpectedTentativeSymbols []string `yaml:"expected_tentative_symbols" json:"expected_tentative_symbols,omitempty"` - Difficulty string `yaml:"difficulty" json:"difficulty,omitempty"` -} - -// Corpus holds the collection of benchmark queries. -// @intent group benchmark queries into a reusable corpus that can be run and validated together. -type Corpus struct { - Version string `yaml:"version" json:"version,omitempty"` - Queries []Query `yaml:"queries" json:"queries"` -} - -// ToolCall records a single tool invocation during query execution. -// @intent capture the tool usage footprint of one benchmarked query execution. -type ToolCall struct { - Tool string `json:"tool"` - ToolUseID string `json:"tool_use_id,omitempty"` - Input string `json:"input,omitempty"` - Output string `json:"output,omitempty"` -} - -// RunResult captures the outcome of executing a single query. -// @intent store answer text, tool usage, timing, and token counts for one benchmark query. -type RunResult struct { - QueryID string `json:"query_id"` - ToolCalls []ToolCall `json:"tool_calls,omitempty"` - FilesRead []string `json:"files_read,omitempty"` - Answer string `json:"answer,omitempty"` - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - ElapsedMs int64 `json:"elapsed_ms"` - Error string `json:"error,omitempty"` -} - -// BenchmarkRun holds all results from a single benchmark execution. -// @intent represent one complete benchmark session for a given execution mode. -type BenchmarkRun struct { - Mode string `json:"mode"` - RunAt time.Time `json:"run_at"` - Results []RunResult `json:"results"` -} - -// ResultByID returns the RunResult for the given query ID, or nil if not found. -// @intent support direct lookup of a query result inside a benchmark run. -func (r *BenchmarkRun) ResultByID(id string) *RunResult { - for i := range r.Results { - if r.Results[i].QueryID == id { - return &r.Results[i] - } - } - return nil -} - -// MatchResult holds the computed match metrics for a single query. -// @intent summarize scored file, symbol, tool, and token metrics for one benchmark query. -type MatchResult struct { - QueryID string `json:"query_id"` - FileHitRatio float64 `json:"file_hit_ratio"` - SymbolHitRatio float64 `json:"symbol_hit_ratio"` - StrictSymbolHitRatio float64 `json:"strict_symbol_hit_ratio"` - TentativeSymbolHitRatio float64 `json:"tentative_symbol_hit_ratio"` - LLMStrictBias float64 `json:"llm_strict_bias"` - StrictContaminationRate float64 `json:"strict_contamination_rate"` - ToolAwareStrictRatio float64 `json:"tool_aware_strict_ratio"` - ToolAwareTentativeRatio float64 `json:"tool_aware_tentative_ratio"` - TotalToolCalls int `json:"total_tool_calls"` - CcgToolCalls int `json:"ccg_tool_calls"` - TotalInputTokens int `json:"total_input_tokens"` -} - -// ComparisonReport holds a comparison between two benchmark runs. -// @intent package benchmark runs and their scored matches into one comparison artifact. -type ComparisonReport struct { - WithCCG *BenchmarkRun `json:"with_ccg"` - WithoutCCG *BenchmarkRun `json:"without_ccg,omitempty"` - Matches []MatchResult `json:"matches"` - MatchesWithout []MatchResult `json:"matches_without,omitempty"` -} diff --git a/internal/benchmark/schema_test.go b/internal/benchmark/schema_test.go deleted file mode 100644 index 1f1ba17..0000000 --- a/internal/benchmark/schema_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package benchmark_test - -import ( - "encoding/json" - "testing" - "time" - - "go.yaml.in/yaml/v3" - - "github.com/tae2089/code-context-graph/internal/benchmark" -) - -func TestQuery_YAMLRoundtrip(t *testing.T) { - q := benchmark.Query{ - ID: "q1", - Description: "결제 실패 복구 로직", - ExpectedFiles: []string{"internal/webhook/handler.go"}, - ExpectedSymbols: []string{"WebhookHandler"}, - Difficulty: "medium", - } - data, err := yaml.Marshal(q) - if err != nil { - t.Fatalf("marshal: %v", err) - } - var got benchmark.Query - if err := yaml.Unmarshal(data, &got); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if got.ID != q.ID { - t.Errorf("ID: got %q, want %q", got.ID, q.ID) - } - if got.Description != q.Description { - t.Errorf("Description: got %q, want %q", got.Description, q.Description) - } - if len(got.ExpectedFiles) != 1 || got.ExpectedFiles[0] != q.ExpectedFiles[0] { - t.Errorf("ExpectedFiles: got %v, want %v", got.ExpectedFiles, q.ExpectedFiles) - } - if got.Difficulty != q.Difficulty { - t.Errorf("Difficulty: got %q, want %q", got.Difficulty, q.Difficulty) - } -} - -func TestRunResult_JSONRoundtrip(t *testing.T) { - r := benchmark.RunResult{ - QueryID: "q1", - ToolCalls: []benchmark.ToolCall{{Tool: "mcp__ccg__search", Input: "query"}}, - FilesRead: []string{"internal/webhook/handler.go"}, - Answer: "WebhookHandler는 ...", - InputTokens: 100, - OutputTokens: 50, - ElapsedMs: 1234, - } - data, err := json.Marshal(r) - if err != nil { - t.Fatalf("marshal: %v", err) - } - var got benchmark.RunResult - if err := json.Unmarshal(data, &got); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if got.QueryID != r.QueryID { - t.Errorf("QueryID: got %q, want %q", got.QueryID, r.QueryID) - } - if len(got.ToolCalls) != 1 || got.ToolCalls[0].Tool != r.ToolCalls[0].Tool { - t.Errorf("ToolCalls: got %v, want %v", got.ToolCalls, r.ToolCalls) - } - if got.InputTokens != r.InputTokens { - t.Errorf("InputTokens: got %d, want %d", got.InputTokens, r.InputTokens) - } - if got.ElapsedMs != r.ElapsedMs { - t.Errorf("ElapsedMs: got %d, want %d", got.ElapsedMs, r.ElapsedMs) - } -} - -func TestBenchmarkRun_ResultByID(t *testing.T) { - run := benchmark.BenchmarkRun{ - Mode: "with-ccg", - RunAt: time.Now(), - Results: []benchmark.RunResult{ - {QueryID: "q1", Answer: "답변1"}, - {QueryID: "q2", Answer: "답변2"}, - }, - } - got := run.ResultByID("q1") - if got == nil { - t.Fatal("expected non-nil result for q1") - } - if got.Answer != "답변1" { - t.Errorf("Answer: got %q, want %q", got.Answer, "답변1") - } -} - -func TestBenchmarkRun_ResultByID_Missing(t *testing.T) { - run := benchmark.BenchmarkRun{ - Mode: "with-ccg", - Results: []benchmark.RunResult{{QueryID: "q1"}}, - } - got := run.ResultByID("not-exist") - if got != nil { - t.Errorf("expected nil for missing ID, got %+v", got) - } -} diff --git a/internal/benchmark/token_bench.go b/internal/benchmark/token_bench.go deleted file mode 100644 index 6c21948..0000000 --- a/internal/benchmark/token_bench.go +++ /dev/null @@ -1,346 +0,0 @@ -// @index Token baseline and graph-assisted retrieval benchmarking helpers. -package benchmark - -import ( - "context" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "gorm.io/gorm" - - "github.com/tae2089/code-context-graph/internal/model" - "github.com/tae2089/code-context-graph/internal/pathutil" -) - -// DefaultNaiveTokensMaxFileBytes caps per-file size considered by the naive baseline. -// @intent prevent oversized files from skewing or dominating the naive token baseline. -const DefaultNaiveTokensMaxFileBytes int64 = 1 << 20 - -// NaiveTokensOptions tunes which files contribute to the naive token baseline. -// @intent control file selection (excludes, size cap) for the naive reading baseline. -type NaiveTokensOptions struct { - Excludes []string - MaxFileBytes int64 -} - -// SearchBackend는 FTS 검색 백엔드 추상화다. -// @intent abstract graph search so token benchmarks can run against any configured backend. -type SearchBackend interface { - Query(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) -} - -// NodeExpander는 노드의 1-hop 이웃과 어노테이션을 가져오는 추상화다. -// nil을 전달하면 확장 없이 기본 검색 결과만 사용한다. -// @intent optionally enrich graph token context with neighbors and annotations beyond raw search hits. -type NodeExpander interface { - GetEdgesFrom(ctx context.Context, nodeID uint) ([]model.Edge, error) - GetNodesByIDs(ctx context.Context, ids []uint) ([]model.Node, error) - GetAnnotation(ctx context.Context, nodeID uint) (*model.Annotation, error) -} - -// EstimateTokens는 텍스트의 대략적인 토큰 수를 반환한다 (4자 = 1토큰). -// @intent provide a cheap, consistent token estimate for baseline-versus-graph comparisons. -func EstimateTokens(text string) int { - return len(text) / 4 -} - -// NaiveTokens는 repoRoot 아래 exts 확장자를 가진 모든 파일의 토큰 수 합계를 반환한다. -// @intent measure the naive full-file reading baseline for a repository slice. -func NaiveTokens(repoRoot string, exts []string) (int, error) { - return NaiveTokensWithOptions(repoRoot, exts, NaiveTokensOptions{}) -} - -// NaiveTokensWithOptions는 repoRoot 아래 exts 확장자를 가진 파일 중 옵션에 맞는 파일만 집계한다. -// @intent compute a configurable naive token baseline while skipping excluded or oversized files. -func NaiveTokensWithOptions(repoRoot string, exts []string, opts NaiveTokensOptions) (int, error) { - extSet := make(map[string]struct{}, len(exts)) - for _, e := range exts { - extSet[e] = struct{}{} - } - maxFileBytes := opts.MaxFileBytes - if maxFileBytes <= 0 { - maxFileBytes = DefaultNaiveTokensMaxFileBytes - } - var total int - err := filepath.WalkDir(repoRoot, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if pathutil.ShouldSkipDir(d.Name()) { - return filepath.SkipDir - } - return nil - } - relPath, relErr := filepath.Rel(repoRoot, path) - if relErr != nil { - return relErr - } - if pathutil.MatchExcludes(opts.Excludes, relPath) { - return nil - } - if _, ok := extSet[filepath.Ext(path)]; !ok { - return nil - } - info, statErr := d.Info() - if statErr != nil { - return nil - } - if info.Size() > maxFileBytes { - return nil - } - data, rerr := os.ReadFile(path) - if rerr != nil { - return nil - } - total += EstimateTokens(string(data)) - return nil - }) - return total, err -} - -// RunTokenBenchOptions bundles optional knobs for the token benchmark execution. -// @intent extend the token benchmark with baseline tuning while keeping the core API stable. -type RunTokenBenchOptions struct { - Naive NaiveTokensOptions -} - -// TokenBenchResult는 단일 쿼리에 대한 토큰 벤치마크 결과다. -// @intent report naive-versus-graph token cost and recall for one benchmark query. -type TokenBenchResult struct { - QueryID string `json:"query_id"` - NaiveTokens int `json:"naive_tokens"` - GraphTokens int `json:"graph_tokens"` - Ratio float64 `json:"ratio"` - SearchElapsedMs int64 `json:"search_elapsed_ms"` - ResultCount int `json:"result_count"` - // Recall: 정답 파일/심볼이 결과에 포함되었는지 측정 - FilesHit int `json:"files_hit"` - FilesTotal int `json:"files_total"` - SymbolsHit int `json:"symbols_hit"` - SymbolsTotal int `json:"symbols_total"` - Recall float64 `json:"recall"` -} - -// extractASCIITerms는 텍스트에서 ASCII 영숫자 단어만 추출한다. -// 한국어 등 비ASCII 문자를 포함한 설명에서 코드 심볼에 해당하는 영어 단어만 반환한다. -// @intent recover code-like search terms from natural-language benchmark descriptions. -func extractASCIITerms(text string) []string { - var terms []string - for _, word := range strings.Fields(text) { - var clean strings.Builder - for _, r := range word { - if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { - clean.WriteRune(r) - } - } - if clean.Len() > 1 { - terms = append(terms, clean.String()) - } - } - return terms -} - -// readLines는 파일에서 startLine~endLine 범위의 텍스트를 반환한다 (1-based). -// @intent capture a node's code snippet when estimating graph-assisted context size. -func readLines(path string, startLine, endLine int) string { - data, err := os.ReadFile(path) - if err != nil { - return "" - } - lines := strings.Split(string(data), "\n") - lo := max(startLine-1, 0) - hi := min(endLine, len(lines)) - if lo >= hi { - return "" - } - return strings.Join(lines[lo:hi], "\n") -} - -// searchAndCollect는 FTS 검색 후 1-hop 확장까지 수행해 노드 목록과 텍스트를 반환한다. -// seen은 호출 간 중복 제거에 사용되며 nil이면 새로 생성한다. -// @intent gather graph-derived context text and nodes for one token benchmark query. -func searchAndCollect( - ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, - query, repoRoot string, limit int, seen map[uint]struct{}, -) (nodes []model.Node, text string, elapsedMs int64, err error) { - if seen == nil { - seen = make(map[uint]struct{}) - } - - terms := extractASCIITerms(query) - if len(terms) == 0 { - return nil, "", 0, nil - } - - // 단어 수에 반비례해 단어당 limit을 조정한다 (총 결과 예산 유지). - limitPerTerm := max(3, limit/len(terms)) - - var sb strings.Builder - writeNode := func(n model.Node) { - nodes = append(nodes, n) - fmt.Fprintf(&sb, "%s %s %s\n", n.QualifiedName, n.Kind, n.FilePath) - if repoRoot != "" && n.FilePath != "" && n.StartLine > 0 { - code := readLines(filepath.Join(repoRoot, n.FilePath), n.StartLine, n.EndLine) - sb.WriteString(code) - sb.WriteByte('\n') - } - } - - // 각 단어를 개별 검색해 FTS5 AND 제약을 피하고 결과를 누적한다. - for _, term := range terms { - start := time.Now() - found, qerr := backend.Query(ctx, db, term, limitPerTerm) - elapsedMs += time.Since(start).Milliseconds() - if qerr != nil { - return nil, "", elapsedMs, qerr - } - for _, n := range found { - if n.ID > 0 { - if _, dup := seen[n.ID]; dup { - continue - } - seen[n.ID] = struct{}{} - } - writeNode(n) - if expander == nil { - continue - } - edges, eerr := expander.GetEdgesFrom(ctx, n.ID) - if eerr != nil { - continue - } - neighborIDs := make([]uint, 0, len(edges)) - for _, e := range edges { - if _, dup := seen[e.ToNodeID]; !dup { - neighborIDs = append(neighborIDs, e.ToNodeID) - } - } - if len(neighborIDs) == 0 { - continue - } - neighbors, nerr := expander.GetNodesByIDs(ctx, neighborIDs) - if nerr != nil { - continue - } - for _, nb := range neighbors { - seen[nb.ID] = struct{}{} - writeNode(nb) - } - if ann, aerr := expander.GetAnnotation(ctx, n.ID); aerr == nil && ann != nil { - fmt.Fprintf(&sb, "annotation: %s\n", ann.RawText) - } - } - } - return nodes, sb.String(), elapsedMs, nil -} - -// GraphTokens는 단일 쿼리로 검색해 토큰 수, 경과 시간(ms), 결과 수를 반환한다. -// @intent measure the token footprint of graph-assisted retrieval for one query. -func GraphTokens(ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, query, repoRoot string, limit int) (tokens int, elapsedMs int64, count int, err error) { - nodes, text, elapsedMs, err := searchAndCollect(ctx, db, backend, expander, query, repoRoot, limit, nil) - if err != nil { - return 0, elapsedMs, 0, err - } - return EstimateTokens(text), elapsedMs, len(nodes), nil -} - -// countFilesHit는 nodes 중 expectedFiles에 해당하는 FilePath를 가진 노드 수를 반환한다. -// @intent measure how many expected files appear in the retrieved graph context. -func countFilesHit(nodes []model.Node, expectedFiles []string) int { - if len(expectedFiles) == 0 { - return 0 - } - found := make(map[string]struct{}, len(nodes)) - for _, n := range nodes { - found[n.FilePath] = struct{}{} - } - hit := 0 - for _, f := range expectedFiles { - if _, ok := found[f]; ok { - hit++ - } - } - return hit -} - -// countSymbolsHit는 nodes 중 QualifiedName에 expectedSymbols 심볼명이 포함된 노드 수를 반환한다. -// @intent measure how many expected symbols appear in the retrieved graph context. -func countSymbolsHit(nodes []model.Node, expectedSymbols []string) int { - if len(expectedSymbols) == 0 { - return 0 - } - hit := 0 - for _, sym := range expectedSymbols { - for _, n := range nodes { - if strings.Contains(n.QualifiedName, sym) { - hit++ - break - } - } - } - return hit -} - -// computeRecall은 파일/심볼 히트율을 0~1로 반환한다. -// @intent collapse file and symbol hit counts into one recall-style coverage signal. -func computeRecall(filesHit, filesTotal, symbolsHit, symbolsTotal int) float64 { - total := filesTotal + symbolsTotal - if total == 0 { - return 0 - } - return float64(filesHit+symbolsHit) / float64(total) -} - -// RunTokenBench는 corpus의 각 쿼리에 대해 naive/graph 토큰과 recall을 비교한다. -// 검색은 항상 Description을 사용하며, expected_symbols/files는 정답 매칭에만 사용한다. -// limit은 쿼리당 총 결과 예산이며 단어 수에 반비례해 단어당 limit이 자동 조정된다. -// @intent execute the full token benchmark corpus and compare naive reading cost against graph retrieval cost. -func RunTokenBench(ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, corpus *Corpus, repoRoot string, exts []string, limit int) ([]TokenBenchResult, error) { - return RunTokenBenchWithOptions(ctx, db, backend, expander, corpus, repoRoot, exts, limit, RunTokenBenchOptions{}) -} - -// @intent extend token benchmarking with baseline tuning options while preserving the core execution flow. -func RunTokenBenchWithOptions(ctx context.Context, db *gorm.DB, backend SearchBackend, expander NodeExpander, corpus *Corpus, repoRoot string, exts []string, limit int, opts RunTokenBenchOptions) ([]TokenBenchResult, error) { - naive, err := NaiveTokensWithOptions(repoRoot, exts, opts.Naive) - if err != nil { - return nil, err - } - - results := make([]TokenBenchResult, 0, len(corpus.Queries)) - for _, q := range corpus.Queries { - nodes, text, elapsed, qerr := searchAndCollect(ctx, db, backend, expander, q.Description, repoRoot, limit, nil) - if qerr != nil { - return nil, qerr - } - tokens := EstimateTokens(text) - - filesHit := countFilesHit(nodes, q.ExpectedFiles) - symbolsHit := countSymbolsHit(nodes, q.ExpectedSymbols) - filesTotal := len(q.ExpectedFiles) - symbolsTotal := len(q.ExpectedSymbols) - - var ratio float64 - if tokens > 0 { - ratio = float64(naive) / float64(tokens) - } - results = append(results, TokenBenchResult{ - QueryID: q.ID, - NaiveTokens: naive, - GraphTokens: tokens, - Ratio: ratio, - SearchElapsedMs: elapsed, - ResultCount: len(nodes), - FilesHit: filesHit, - FilesTotal: filesTotal, - SymbolsHit: symbolsHit, - SymbolsTotal: symbolsTotal, - Recall: computeRecall(filesHit, filesTotal, symbolsHit, symbolsTotal), - }) - } - return results, nil -} diff --git a/internal/benchmark/token_bench_test.go b/internal/benchmark/token_bench_test.go deleted file mode 100644 index a24c3df..0000000 --- a/internal/benchmark/token_bench_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package benchmark - -import ( - "context" - "os" - "path/filepath" - "testing" - - "gorm.io/gorm" - - "github.com/tae2089/code-context-graph/internal/model" -) - -// mockSearchBackend는 테스트용 검색 백엔드다. -type mockSearchBackend struct { - nodes []model.Node -} - -func (m *mockSearchBackend) Query(_ context.Context, _ *gorm.DB, _ string, _ int) ([]model.Node, error) { - return m.nodes, nil -} - -// mockNodeExpander는 테스트용 노드 확장기다. -type mockNodeExpander struct { - edges map[uint][]model.Edge - nodes map[uint]model.Node -} - -func (m *mockNodeExpander) GetEdgesFrom(_ context.Context, nodeID uint) ([]model.Edge, error) { - return m.edges[nodeID], nil -} - -func (m *mockNodeExpander) GetNodesByIDs(_ context.Context, ids []uint) ([]model.Node, error) { - var result []model.Node - for _, id := range ids { - if n, ok := m.nodes[id]; ok { - result = append(result, n) - } - } - return result, nil -} - -func (m *mockNodeExpander) GetAnnotation(_ context.Context, _ uint) (*model.Annotation, error) { - return nil, nil -} - -func TestEstimateTokens_FourCharsPerToken(t *testing.T) { - if got := EstimateTokens("abcd"); got != 1 { - t.Errorf("want 1, got %d", got) - } - if got := EstimateTokens("abcdefgh"); got != 2 { - t.Errorf("want 2, got %d", got) - } - if got := EstimateTokens(""); got != 0 { - t.Errorf("want 0, got %d", got) - } -} - -func TestNaiveTokens_CountsSourceFiles(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "util.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - // .txt는 집계 제외 - if err := os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("abcdabcdabcd"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokens(dir, []string{".go"}) - if err != nil { - t.Fatal(err) - } - // main.go: 1, util.go: 2 → 합계 3 - if got != 3 { - t.Errorf("want 3, got %d", got) - } -} - -func TestNaiveTokens_EmptyDir(t *testing.T) { - dir := t.TempDir() - got, err := NaiveTokens(dir, []string{".go"}) - if err != nil { - t.Fatal(err) - } - if got != 0 { - t.Errorf("want 0, got %d", got) - } -} - -func TestNaiveTokensWithOptions_SkipsDefaultDirs(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "vendor"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "vendor", "skip.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokensWithOptions(dir, []string{".go"}, NaiveTokensOptions{}) - if err != nil { - t.Fatal(err) - } - if got != 1 { - t.Fatalf("want only main.go counted, got %d", got) - } -} - -func TestNaiveTokensWithOptions_RespectsExcludes(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "gen"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "gen", "skip.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokensWithOptions(dir, []string{".go"}, NaiveTokensOptions{Excludes: []string{"gen"}}) - if err != nil { - t.Fatal(err) - } - if got != 1 { - t.Fatalf("want only main.go counted after exclude, got %d", got) - } -} - -func TestNaiveTokensWithOptions_SkipsLargeFiles(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "small.go"), []byte("abcd"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "large.go"), []byte("abcdefgh"), 0o600); err != nil { - t.Fatal(err) - } - - got, err := NaiveTokensWithOptions(dir, []string{".go"}, NaiveTokensOptions{MaxFileBytes: 4}) - if err != nil { - t.Fatal(err) - } - if got != 1 { - t.Fatalf("want only small.go counted under size cap, got %d", got) - } -} - -func TestGraphTokens_SumNodeText(t *testing.T) { - backend := &mockSearchBackend{ - nodes: []model.Node{ - {QualifiedName: "pkg.Foo", Kind: model.NodeKindFunction, FilePath: "foo.go"}, - {QualifiedName: "pkg.Bar", Kind: model.NodeKindFunction, FilePath: "bar.go"}, - }, - } - tokens, elapsed, count, err := GraphTokens(context.Background(), nil, backend, nil, "foo", "", 10) - if err != nil { - t.Fatal(err) - } - if count != 2 { - t.Errorf("want count=2, got %d", count) - } - if tokens <= 0 { - t.Errorf("want tokens>0, got %d", tokens) - } - if elapsed < 0 { - t.Errorf("want elapsed>=0, got %d", elapsed) - } -} - -func TestGraphTokens_WithExpander_IncludesNeighbors(t *testing.T) { - searchBackend := &mockSearchBackend{ - nodes: []model.Node{ - {ID: 1, QualifiedName: "pkg.Foo", Kind: model.NodeKindFunction, FilePath: "foo.go"}, - }, - } - expander := &mockNodeExpander{ - edges: map[uint][]model.Edge{ - 1: {{FromNodeID: 1, ToNodeID: 2, Kind: model.EdgeKindCalls}}, - }, - nodes: map[uint]model.Node{ - 2: {ID: 2, QualifiedName: "pkg.Bar", Kind: model.NodeKindFunction, FilePath: "bar.go"}, - }, - } - - tokensWithout, _, _, err := GraphTokens(context.Background(), nil, searchBackend, nil, "foo", "", 10) - - if err != nil { - t.Fatal(err) - } - tokensWith, _, _, err := GraphTokens(context.Background(), nil, searchBackend, expander, "foo", "", 10) - if err != nil { - t.Fatal(err) - } - if tokensWith <= tokensWithout { - t.Errorf("want tokensWith(%d) > tokensWithout(%d)", tokensWith, tokensWithout) - } -} - -func TestRunTokenBench_RatioCalculated(t *testing.T) { - dir := t.TempDir() - // 400자 = 100 tokens - content := make([]byte, 400) - for i := range content { - content[i] = 'a' - } - if err := os.WriteFile(filepath.Join(dir, "main.go"), content, 0o600); err != nil { - t.Fatal(err) - } - - backend := &mockSearchBackend{ - nodes: []model.Node{ - {QualifiedName: "pkg.Foo", Kind: model.NodeKindFunction, FilePath: "foo.go"}, - }, - } - corpus := &Corpus{ - Queries: []Query{ - {ID: "q1", Description: "foo 함수 설명"}, - }, - } - - results, err := RunTokenBench(context.Background(), nil, backend, nil, corpus, dir, []string{".go"}, 10) - if err != nil { - t.Fatal(err) - } - if len(results) != 1 { - t.Fatalf("want 1 result, got %d", len(results)) - } - r := results[0] - if r.QueryID != "q1" { - t.Errorf("want query_id=q1, got %s", r.QueryID) - } - if r.NaiveTokens != 100 { - t.Errorf("want naive_tokens=100, got %d", r.NaiveTokens) - } - if r.GraphTokens <= 0 { - t.Errorf("want graph_tokens>0, got %d", r.GraphTokens) - } - if r.Ratio <= 0 { - t.Errorf("want ratio>0, got %f", r.Ratio) - } -} diff --git a/internal/cli/benchmark.go b/internal/cli/benchmark.go deleted file mode 100644 index 17dd122..0000000 --- a/internal/cli/benchmark.go +++ /dev/null @@ -1,334 +0,0 @@ -// @index CLI commands for benchmarking CCG-assisted workflows and token savings. -package cli - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/viper" - "github.com/tae2089/trace" - - "github.com/tae2089/code-context-graph/internal/benchmark" - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/store/gormstore" - storesearch "github.com/tae2089/code-context-graph/internal/store/search" -) - -// @intent group benchmark subcommands that measure retrieval quality and token reduction workflows. -func newBenchmarkCmd(deps *Deps) *cobra.Command { - cmd := &cobra.Command{ - Use: "benchmark", - Short: "Benchmark ccg MCP tool effectiveness", - } - cmd.AddCommand( - newBenchmarkInitCmd(), - newBenchmarkValidateCmd(), - newBenchmarkRunCmd(), - newBenchmarkAnalyzeCmd(), - newBenchmarkCompareCmd(), - newBenchmarkReportCmd(), - newBenchmarkTokenBenchCmd(deps), - ) - return cmd -} - -// newBenchmarkInitCmd creates a corpus directory with a template queries.yaml. -// @intent bootstrap a benchmark corpus so users can start with a valid query fixture quickly. -func newBenchmarkInitCmd() *cobra.Command { - var outDir string - cmd := &cobra.Command{ - Use: "init", - Short: "Initialize a benchmark corpus directory with a template queries.yaml", - RunE: func(cmd *cobra.Command, _ []string) error { - if err := os.MkdirAll(outDir, 0o755); err != nil { - return trace.Wrap(err, "create corpus directory") - } - corpus := &benchmark.Corpus{ - Version: "1", - Queries: []benchmark.Query{ - { - ID: "example-01", - Description: "웹훅 push 수신 후 자동 그래프 업데이트 흐름", - ExpectedFiles: []string{"internal/webhook/handler.go"}, - ExpectedSymbols: []string{"WebhookHandler"}, - Difficulty: "medium", - }, - }, - } - path := filepath.Join(outDir, "queries.yaml") - if err := benchmark.SaveCorpus(path, corpus); err != nil { - return trace.Wrap(err, "write template corpus") - } - fmt.Fprintf(stdout(cmd), "Initialized benchmark corpus at %s\n", path) - return nil - }, - } - cmd.Flags().StringVar(&outDir, "out", "testdata/benchmark", "Output directory for corpus files") - return cmd -} - -// newBenchmarkValidateCmd validates a queries.yaml file. -// @intent fail fast on malformed benchmark corpora before expensive benchmark runs start. -func newBenchmarkValidateCmd() *cobra.Command { - var corpusPath string - cmd := &cobra.Command{ - Use: "validate", - Short: "Validate a benchmark corpus YAML file", - RunE: func(cmd *cobra.Command, _ []string) error { - if _, err := benchmark.LoadCorpus(corpusPath); err != nil { - return trace.Wrap(err, "validate corpus") - } - fmt.Fprintf(stdout(cmd), "Corpus %s is valid\n", corpusPath) - return nil - }, - } - cmd.Flags().StringVar(&corpusPath, "corpus", "testdata/benchmark/queries.yaml", "Path to corpus YAML file") - return cmd -} - -// newBenchmarkRunCmd executes each query via `claude -p` subprocess. -// @intent run the benchmark corpus through Claude CLI and capture structured results for later comparison. -func newBenchmarkRunCmd() *cobra.Command { - var corpusPath, cwd, mode, outPath string - cmd := &cobra.Command{ - Use: "run", - Short: "Run benchmark queries via claude CLI subprocess", - RunE: func(cmd *cobra.Command, _ []string) error { - if cwd == "" { - return fmt.Errorf("--cwd is required: set the benchmark working directory") - } - corpus, err := benchmark.LoadCorpus(corpusPath) - if err != nil { - return trace.Wrap(err, "load corpus") - } - cfg := benchmark.DefaultRunnerConfig() - cfg.Mode = mode - cfg.CWD = cwd - runner := benchmark.NewRunner(cfg, &osExecutor{}) - run, err := runner.Run(cmd.Context(), corpus) - if err != nil { - return trace.Wrap(err, "run benchmark") - } - data, err := json.MarshalIndent(run, "", " ") - if err != nil { - return trace.Wrap(err, "marshal run") - } - if outPath != "" { - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return trace.Wrap(err, "write output") - } - fmt.Fprintf(stdout(cmd), "Results saved to %s\n", outPath) - } else { - fmt.Fprintln(stdout(cmd), string(data)) - } - return nil - }, - } - cmd.Flags().StringVar(&corpusPath, "corpus", "testdata/benchmark/queries.yaml", "Path to corpus YAML file") - cmd.Flags().StringVar(&cwd, "cwd", "", "Benchmark working directory (required)") - cmd.Flags().StringVar(&mode, "mode", "with-ccg", "Benchmark mode: with-ccg or without-ccg") - cmd.Flags().StringVar(&outPath, "out", "", "Output file path (JSON); defaults to stdout") - return cmd -} - -// newBenchmarkAnalyzeCmd parses a Claude Code session JSONL and extracts RunResults. -// @intent recover benchmark run results from recorded Claude Code session logs. -func newBenchmarkAnalyzeCmd() *cobra.Command { - var sessionPath string - cmd := &cobra.Command{ - Use: "analyze", - Short: "Parse a Claude Code session JSONL and extract query run results", - RunE: func(cmd *cobra.Command, _ []string) error { - if sessionPath == "" { - return fmt.Errorf("--session is required: path to Claude Code session JSONL file") - } - msgs, err := benchmark.ParseJSONL(sessionPath) - if err != nil { - return trace.Wrap(err, "parse JSONL") - } - segs, err := benchmark.ExtractQuerySegments(msgs) - if err != nil { - return trace.Wrap(err, "extract segments") - } - results := make([]benchmark.RunResult, 0, len(segs)) - for _, seg := range segs { - results = append(results, benchmark.ExtractRunResult(seg.QueryID, seg)) - } - data, err := json.MarshalIndent(results, "", " ") - if err != nil { - return trace.Wrap(err, "marshal results") - } - fmt.Fprintln(stdout(cmd), string(data)) - return nil - }, - } - cmd.Flags().StringVar(&sessionPath, "session", "", "Path to Claude Code session JSONL file (required)") - return cmd -} - -// newBenchmarkCompareCmd compares two BenchmarkRun JSON files. -// @intent quantify the delta between with-ccg and without-ccg benchmark runs. -func newBenchmarkCompareCmd() *cobra.Command { - var withPath, withoutPath, corpusPath string - cmd := &cobra.Command{ - Use: "compare", - Short: "Compare two benchmark runs (with-ccg vs without-ccg)", - RunE: func(cmd *cobra.Command, _ []string) error { - withRun, err := loadBenchmarkRun(withPath) - if err != nil { - return trace.Wrap(err, "load with-ccg run") - } - var withoutRun *benchmark.BenchmarkRun - if withoutPath != "" { - withoutRun, err = loadBenchmarkRun(withoutPath) - if err != nil { - return trace.Wrap(err, "load without-ccg run") - } - } - var corpus *benchmark.Corpus - if corpusPath != "" { - corpus, err = benchmark.LoadCorpus(corpusPath) - if err != nil { - return trace.Wrap(err, "load corpus") - } - } else { - corpus = &benchmark.Corpus{} - } - report := benchmark.Compare(withRun, withoutRun, corpus) - data, err := json.MarshalIndent(report, "", " ") - if err != nil { - return trace.Wrap(err, "marshal report") - } - fmt.Fprintln(stdout(cmd), string(data)) - return nil - }, - } - cmd.Flags().StringVar(&withPath, "with", "", "Path to with-ccg run JSON (required)") - cmd.Flags().StringVar(&withoutPath, "without", "", "Path to without-ccg run JSON (optional)") - cmd.Flags().StringVar(&corpusPath, "corpus", "", "Path to corpus YAML for expected matching") - _ = cmd.MarkFlagRequired("with") - return cmd -} - -// newBenchmarkReportCmd generates a markdown report from a ComparisonReport JSON. -// @intent turn raw benchmark comparison output into a shareable markdown summary. -func newBenchmarkReportCmd() *cobra.Command { - var compPath, outPath string - cmd := &cobra.Command{ - Use: "report", - Short: "Generate a markdown report from a comparison JSON file", - RunE: func(cmd *cobra.Command, _ []string) error { - if compPath == "" { - return fmt.Errorf("--comparison is required") - } - data, err := os.ReadFile(compPath) - if err != nil { - return trace.Wrap(err, "read comparison") - } - var report benchmark.ComparisonReport - if err := json.Unmarshal(data, &report); err != nil { - return trace.Wrap(err, "parse comparison") - } - if err := benchmark.WriteReport(&report, outPath); err != nil { - return trace.Wrap(err, "write report") - } - fmt.Fprintf(stdout(cmd), "Report written to %s\n", outPath) - return nil - }, - } - cmd.Flags().StringVar(&compPath, "comparison", "", "Path to comparison JSON file (required)") - cmd.Flags().StringVar(&outPath, "out", "report.md", "Output markdown file path") - return cmd -} - -// loadBenchmarkRun reads and unmarshals a BenchmarkRun from a JSON file. -// @intent centralize benchmark run loading for compare and reporting commands. -func loadBenchmarkRun(path string) (*benchmark.BenchmarkRun, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var run benchmark.BenchmarkRun - if err := json.Unmarshal(data, &run); err != nil { - return nil, err - } - return &run, nil -} - -// newBenchmarkTokenBenchCmd measures token reduction: naive file reading vs CCG graph query. -// @intent measure token savings from CCG-assisted retrieval without involving an LLM response loop. -func newBenchmarkTokenBenchCmd(deps *Deps) *cobra.Command { - var corpusPath, repoRoot, outPath string - var exts []string - var excludePatterns []string - var limit int - cmd := &cobra.Command{ - Use: "token-bench", - Short: "Measure token reduction: naive file reading vs CCG graph query (no LLM)", - RunE: func(cmd *cobra.Command, _ []string) error { - if deps.DB == nil { - return errDBNotInitialized - } - corpus, err := benchmark.LoadCorpus(corpusPath) - if err != nil { - return trace.Wrap(err, "load corpus") - } - backend := storesearch.NewSQLiteBackend() - ctx := ctxns.WithNamespace(cmd.Context(), viper.GetString("namespace")) - results, err := benchmark.RunTokenBenchWithOptions(ctx, deps.DB, backend, gormstore.New(deps.DB), corpus, repoRoot, exts, limit, benchmark.RunTokenBenchOptions{ - Naive: benchmark.NaiveTokensOptions{Excludes: resolveExcludes(excludePatterns)}, - }) - if err != nil { - return trace.Wrap(err, "run token bench") - } - data, err := json.MarshalIndent(results, "", " ") - if err != nil { - return trace.Wrap(err, "marshal results") - } - if outPath != "" { - if err := os.WriteFile(outPath, data, 0o644); err != nil { - return trace.Wrap(err, "write output") - } - fmt.Fprintf(stdout(cmd), "Results saved to %s\n", outPath) - } else { - fmt.Fprintln(stdout(cmd), string(data)) - } - return nil - }, - } - cmd.Flags().StringVar(&corpusPath, "corpus", "testdata/benchmark/queries.yaml", "Path to corpus YAML file") - cmd.Flags().StringVar(&repoRoot, "repo", ".", "Repository root to measure naive tokens") - cmd.Flags().StringVar(&outPath, "out", "", "Output JSON file; defaults to stdout") - cmd.Flags().StringSliceVar(&exts, "exts", []string{".go"}, "Source file extensions to count") - cmd.Flags().StringArrayVar(&excludePatterns, "exclude", nil, "Exclude files/paths from naive token baseline (repeatable)") - cmd.Flags().IntVar(&limit, "limit", 30, "Total result budget per query (auto-split across terms)") - return cmd -} - -// osExecutor implements benchmark.Executor using os/exec to call the `claude` CLI. -// @intent bridge benchmark runner execution to the local Claude CLI process. -type osExecutor struct{} - -// @intent execute one benchmark prompt in a subprocess while preserving CLI-compatible stdin semantics. -func (e *osExecutor) Execute(ctx context.Context, args []string, prompt, dir string) ([]byte, error) { - c := exec.CommandContext(ctx, "claude", args...) - c.Stdin = strings.NewReader(prompt) - if dir != "" { - c.Dir = dir - } - out, err := c.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { - return out, fmt.Errorf("%w: %s", err, exitErr.Stderr) - } - } - return out, err -} diff --git a/internal/cli/benchmark_namespace_test.go b/internal/cli/benchmark_namespace_test.go deleted file mode 100644 index b042fbe..0000000 --- a/internal/cli/benchmark_namespace_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package cli - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/tae2089/code-context-graph/internal/benchmark" - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/model" -) - -func TestBenchmarkTokenBench_RespectsNamespace(t *testing.T) { - deps, stdout, stderr, db := setupSearchTest(t) - - nodeA := model.Node{Namespace: "ns-a", Name: "SharedA", QualifiedName: "pkg.SharedA", Kind: model.NodeKindFunction, FilePath: "a.go", StartLine: 1, EndLine: 1, Language: "go"} - nodeB := model.Node{Namespace: "ns-b", Name: "SharedB", QualifiedName: "pkg.SharedB", Kind: model.NodeKindFunction, FilePath: "b.go", StartLine: 1, EndLine: 1, Language: "go"} - if err := db.Create(&nodeA).Error; err != nil { - t.Fatalf("create nodeA: %v", err) - } - if err := db.Create(&nodeB).Error; err != nil { - t.Fatalf("create nodeB: %v", err) - } - if err := db.Create(&model.SearchDocument{Namespace: "ns-a", NodeID: nodeA.ID, Content: "sharedterm alpha", Language: "go"}).Error; err != nil { - t.Fatalf("create docA: %v", err) - } - if err := db.Create(&model.SearchDocument{Namespace: "ns-b", NodeID: nodeB.ID, Content: "sharedterm beta", Language: "go"}).Error; err != nil { - t.Fatalf("create docB: %v", err) - } - if err := deps.SearchBackend.Rebuild(ctxns.WithNamespace(context.Background(), "ns-a"), db); err != nil { - t.Fatalf("rebuild ns-a search: %v", err) - } - if err := deps.SearchBackend.Rebuild(ctxns.WithNamespace(context.Background(), "ns-b"), db); err != nil { - t.Fatalf("rebuild ns-b search: %v", err) - } - - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "b.go"), []byte("package p\nfunc SharedB() {}\n"), 0o644); err != nil { - t.Fatalf("write repo file: %v", err) - } - corpusPath := filepath.Join(t.TempDir(), "queries.yaml") - if err := os.WriteFile(corpusPath, []byte(`queries: - - id: q1 - description: sharedterm - expected_symbols: - - SharedB -`), 0o644); err != nil { - t.Fatalf("write corpus: %v", err) - } - - if err := executeCmd(deps, stdout, stderr, "--namespace", "ns-b", "benchmark", "token-bench", "--corpus", corpusPath, "--repo", root, "--exts", ".go"); err != nil { - t.Fatalf("token-bench: %v\nstderr=%s", err, stderr.String()) - } - - var results []benchmark.TokenBenchResult - if err := json.Unmarshal(stdout.Bytes(), &results); err != nil { - t.Fatalf("decode output: %v\nstdout=%s", err, stdout.String()) - } - if len(results) != 1 { - t.Fatalf("results = %d, want 1", len(results)) - } - if results[0].ResultCount == 0 || results[0].SymbolsHit != 1 { - t.Fatalf("token-bench did not use ns-b search results: %+v", results[0]) - } -} diff --git a/internal/cli/benchmark_test.go b/internal/cli/benchmark_test.go deleted file mode 100644 index a3b35a1..0000000 --- a/internal/cli/benchmark_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package cli_test - -import ( - "bytes" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/cli" -) - -// runBenchmarkCmd is a helper that runs `ccg benchmark ` with a fresh Deps. -func runBenchmarkCmd(t *testing.T, args ...string) (stdout string, err error) { - t.Helper() - var buf bytes.Buffer - deps := &cli.Deps{} - root := cli.NewRootCmd(deps) - root.SetOut(&buf) - root.SetErr(&buf) - root.SetArgs(append([]string{"benchmark"}, args...)) - err = root.Execute() - return buf.String(), err -} - -func TestBenchmarkCmd_HasSubcommands(t *testing.T) { - var buf bytes.Buffer - deps := &cli.Deps{} - root := cli.NewRootCmd(deps) - root.SetOut(&buf) - root.SetErr(&buf) - root.SetArgs([]string{"benchmark", "--help"}) - _ = root.Execute() - out := buf.String() - for _, sub := range []string{"init", "validate", "run", "analyze", "compare", "report"} { - if !strings.Contains(out, sub) { - t.Errorf("help output missing subcommand %q:\n%s", sub, out) - } - } -} - -func TestBenchmarkInitCmd_CreatesDir(t *testing.T) { - dir := t.TempDir() - outDir := filepath.Join(dir, "corpus") - _, err := runBenchmarkCmd(t, "init", "--out", outDir) - if err != nil { - t.Fatalf("benchmark init: %v", err) - } - queriesPath := filepath.Join(outDir, "queries.yaml") - if _, err := os.Stat(queriesPath); os.IsNotExist(err) { - t.Errorf("queries.yaml not created at %s", queriesPath) - } -} - -func TestBenchmarkValidateCmd_ValidCorpus(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 - description: "valid query" -` - _ = os.WriteFile(path, []byte(content), 0o644) - _, err := runBenchmarkCmd(t, "validate", "--corpus", path) - if err != nil { - t.Errorf("expected success for valid corpus, got: %v", err) - } -} - -func TestBenchmarkValidateCmd_InvalidCorpus(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "queries.yaml") - content := `queries: - - id: q1 - description: "first" - - id: q1 - description: "duplicate" -` - _ = os.WriteFile(path, []byte(content), 0o644) - _, err := runBenchmarkCmd(t, "validate", "--corpus", path) - if err == nil { - t.Error("expected error for duplicate ID corpus") - } -} - -func TestBenchmarkAnalyzeCmd_MissingSession(t *testing.T) { - _, err := runBenchmarkCmd(t, "analyze") - if err == nil { - t.Error("expected error when --session flag is missing") - } -} - -func TestBenchmarkAnalyzeCmd_ReadsJSONL(t *testing.T) { - dir := t.TempDir() - // Write minimal JSONL with markers - jsonlContent := `{"type":"tool_result","tool_use_id":"x","content":"===BENCHMARK_QUERY_START id=q1==="}` + "\n" + - `{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"answer here"}],"usage":{"input_tokens":10,"output_tokens":5}}}` + "\n" + - `{"type":"tool_result","tool_use_id":"y","content":"===BENCHMARK_QUERY_END id=q1==="}` + "\n" - sessionPath := filepath.Join(dir, "session.jsonl") - _ = os.WriteFile(sessionPath, []byte(jsonlContent), 0o644) - - out, err := runBenchmarkCmd(t, "analyze", "--session", sessionPath) - if err != nil { - t.Fatalf("benchmark analyze: %v", err) - } - // Output should be valid JSON containing query results - var result interface{} - if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &result); err != nil { - t.Errorf("output is not valid JSON: %v\noutput: %s", err, out) - } -} - -func TestBenchmarkCompareCmd_TwoFiles(t *testing.T) { - dir := t.TempDir() - - writeRun := func(name, mode string) string { - path := filepath.Join(dir, name) - data, _ := json.Marshal(map[string]interface{}{ - "mode": mode, - "run_at": "2026-01-01T00:00:00Z", - "results": []interface{}{}, - }) - _ = os.WriteFile(path, data, 0o644) - return path - } - - withPath := writeRun("with.json", "with-ccg") - withoutPath := writeRun("without.json", "without-ccg") - - out, err := runBenchmarkCmd(t, "compare", "--with", withPath, "--without", withoutPath) - if err != nil { - t.Fatalf("benchmark compare: %v", err) - } - var result interface{} - if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &result); err != nil { - t.Errorf("compare output is not valid JSON: %v\noutput: %s", err, out) - } -} - -func TestBenchmarkReportCmd_GeneratesMarkdown(t *testing.T) { - dir := t.TempDir() - compPath := filepath.Join(dir, "comparison.json") - outPath := filepath.Join(dir, "report.md") - - compData, _ := json.Marshal(map[string]interface{}{ - "with_ccg": map[string]interface{}{ - "mode": "with-ccg", - "run_at": "2026-01-01T00:00:00Z", - "results": []interface{}{}, - }, - "matches": []interface{}{}, - }) - _ = os.WriteFile(compPath, compData, 0o644) - - _, err := runBenchmarkCmd(t, "report", "--comparison", compPath, "--out", outPath) - if err != nil { - t.Fatalf("benchmark report: %v", err) - } - if _, err := os.Stat(outPath); os.IsNotExist(err) { - t.Error("report.md was not created") - } -} - -func TestBenchmarkRunCmd_RequiresCWD(t *testing.T) { - dir := t.TempDir() - corpusPath := filepath.Join(dir, "queries.yaml") - _ = os.WriteFile(corpusPath, []byte("queries:\n - id: q1\n description: test\n"), 0o644) - // Run without --cwd should fail - _, err := runBenchmarkCmd(t, "run", "--corpus", corpusPath) - if err == nil { - t.Error("expected error when --cwd is missing") - } -} diff --git a/internal/cli/eval.go b/internal/cli/eval.go deleted file mode 100644 index e5e23d0..0000000 --- a/internal/cli/eval.go +++ /dev/null @@ -1,102 +0,0 @@ -// @index CLI command for parser and search evaluation against golden corpora. -package cli - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/spf13/cobra" - "github.com/tae2089/trace" - - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/eval" - "github.com/tae2089/code-context-graph/internal/model" - "github.com/tae2089/code-context-graph/internal/parse/treesitter" -) - -// @intent expose parser and search evaluation workflows from the CLI with one shared command surface. -func newEvalCmd(deps *Deps) *cobra.Command { - var corpusDir string - var suite string - var format string - var update bool - - cmd := &cobra.Command{ - Use: "eval", - Short: "Evaluate parser accuracy and search quality against golden corpus", - RunE: func(cmd *cobra.Command, args []string) error { - ns, _ := cmd.Flags().GetString("namespace") - ctx := ctxns.WithNamespace(cmd.Context(), ns) - opts := eval.RunOptions{ - CorpusDir: corpusDir, - Suite: suite, - Format: format, - Update: update, - Walkers: extToLangWalkers(deps.Walkers), - Writer: stdout(cmd), - } - - if (suite == "all" || suite == "search") && deps.DB != nil && deps.SearchBackend != nil { - opts.SearchFn = makeSearchFn(ctx, deps) - } - - _, err := eval.Run(ctx, opts) - if err != nil { - return trace.Wrap(err, "eval") - } - return nil - }, - } - - cmd.Flags().StringVar(&corpusDir, "corpus", "testdata/eval", "Path to golden corpus directory") - cmd.Flags().StringVar(&suite, "suite", "all", "Evaluation suite: all, parser, search") - cmd.Flags().StringVar(&format, "format", "table", "Output format: table, json") - cmd.Flags().BoolVar(&update, "update", false, "Update golden files from current parser output (parser suite only)") - - return cmd -} - -// @intent adapt the configured search backend to the eval package's simple search callback contract. -func makeSearchFn(ctx context.Context, deps *Deps) eval.SearchFunc { - return func(query string, limit int) ([]string, error) { - if deps.DB == nil || deps.SearchBackend == nil { - return nil, fmt.Errorf("database not initialized for search eval") - } - nodes, err := deps.SearchBackend.Query(ctx, deps.DB, query, limit) - if err != nil { - return nil, err - } - return nodeToKeys(nodes), nil - } -} - -// extToLangWalkers converts extension-keyed walkers (e.g. ".go") to language-keyed (e.g. "go"). -// @intent normalize parser walker lookup so eval suites can address languages consistently. -func extToLangWalkers(walkers map[string]*treesitter.Walker) map[string]*treesitter.Walker { - if walkers == nil { - return nil - } - extToLang := map[string]string{ - ".go": "go", ".py": "python", ".ts": "typescript", ".tsx": "typescript", - ".java": "java", ".rb": "ruby", ".js": "javascript", ".jsx": "javascript", - ".c": "c", ".cpp": "cpp", ".rs": "rust", ".kt": "kotlin", ".php": "php", - ".lua": "lua", ".luau": "lua", - } - result := make(map[string]*treesitter.Walker) - for ext, w := range walkers { - if lang, ok := extToLang[ext]; ok { - result[lang] = w - } - } - return result -} - -// @intent convert search result nodes into stable eval comparison keys. -func nodeToKeys(nodes []model.Node) []string { - keys := make([]string, len(nodes)) - for i, n := range nodes { - keys[i] = string(n.Kind) + ":" + n.Name + "@" + filepath.Base(n.FilePath) - } - return keys -} diff --git a/internal/cli/eval_test.go b/internal/cli/eval_test.go deleted file mode 100644 index 77bdb9f..0000000 --- a/internal/cli/eval_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package cli - -import ( - "context" - "testing" - - "gorm.io/gorm" - - "github.com/tae2089/code-context-graph/internal/ctxns" - "github.com/tae2089/code-context-graph/internal/model" -) - -func TestNodeToKeys_UsesBaseFilePathForEvalMatching(t *testing.T) { - keys := nodeToKeys([]model.Node{{ - Kind: model.NodeKindFunction, - Name: "GetUser", - FilePath: "go/sample.go", - }}) - - if len(keys) != 1 { - t.Fatalf("keys length = %d, want 1", len(keys)) - } - if got, want := keys[0], "function:GetUser@sample.go"; got != want { - t.Fatalf("key = %q, want %q", got, want) - } -} - -type evalSpySearchBackend struct { - queryFn func(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) -} - -func (s *evalSpySearchBackend) Migrate(db *gorm.DB) error { - return nil -} - -func (s *evalSpySearchBackend) Rebuild(ctx context.Context, db *gorm.DB) error { - return nil -} - -func (s *evalSpySearchBackend) RebuildNodes(ctx context.Context, db *gorm.DB, nodeIDs []uint) error { - return nil -} - -func (s *evalSpySearchBackend) PurgeNamespace(ctx context.Context, db *gorm.DB) error { - return nil -} - -func (s *evalSpySearchBackend) Query(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) { - if s.queryFn != nil { - return s.queryFn(ctx, db, query, limit) - } - return nil, nil -} - -func TestMakeSearchFn_UsesProvidedNamespaceContext(t *testing.T) { - deps, _, _ := newTestDeps() - deps.DB = &gorm.DB{} - deps.SearchBackend = &evalSpySearchBackend{queryFn: func(ctx context.Context, db *gorm.DB, query string, limit int) ([]model.Node, error) { - if got := ctxns.FromContext(ctx); got != "alpha" { - t.Fatalf("query context namespace = %q, want %q", got, "alpha") - } - return []model.Node{{Kind: model.NodeKindFunction, Name: "Foo", FilePath: "foo.go"}}, nil - }} - - searchFn := makeSearchFn(ctxns.WithNamespace(context.Background(), "alpha"), deps) - if _, err := searchFn("foo", 5); err != nil { - t.Fatalf("searchFn returned error: %v", err) - } -} diff --git a/internal/cli/lint.go b/internal/cli/lint.go index 1fbd75a..a0c1321 100644 --- a/internal/cli/lint.go +++ b/internal/cli/lint.go @@ -17,13 +17,6 @@ import ( "github.com/tae2089/code-context-graph/internal/pathutil" ) -// countNonIgnored counts lint issues not covered by an ignore rule in .ccg.yaml. -// @intent strict 모드에서 실제 실패로 간주할 lint 항목만 다시 집계한다. -// @domainRule action: ignore로 선언된 규칙은 strict 실패 수에서 제외한다. -func countNonIgnored(report *docs.LintReport) int { - return countNonIgnoredWithRules(report, configuredLintRules()) -} - // normalizeLintCategory maps legacy or variant category names to canonical forms. // @intent ensure rule matching uses consistent category keys regardless of input spelling // @domainRule "deadref" and "drifted" are accepted aliases for "dead-ref" and "drift" @@ -261,12 +254,6 @@ func lintRuleFromMapLookup(lookup func(string) (any, bool)) (lintRule, bool) { return rule, rule.Pattern != "" || rule.Category != "" || rule.Action != "" } -// lintRuleDoc is the top-level YAML structure for a standalone rules file. -// @intent represent the on-disk format used when rules are stored outside .ccg.yaml -type lintRuleDoc struct { - Rules []lintRule `yaml:"rules"` -} - // lintRule is a single suppression or warn rule entry in .ccg.yaml or auto-rules.yaml. // @intent carry the pattern, category, and action that determine how a lint finding is handled // @domainRule auto:true marks rules generated by the Twice Rule engine, not written by the user diff --git a/internal/cli/root.go b/internal/cli/root.go index bb31835..948e479 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -161,9 +161,7 @@ func NewRootCmd(deps *Deps) *cobra.Command { newIndexCmd(deps), newLintCmd(deps), newRagIndexCmd(deps), - newEvalCmd(deps), newVersionCmd(deps), - newBenchmarkCmd(deps), ) return rootCmd diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index af95dea..f537ee4 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -8,7 +8,6 @@ import ( "path/filepath" "testing" - "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -225,15 +224,6 @@ func TestRoot_HooksInstallSkipsInitFuncViaParentAnnotation(t *testing.T) { } } -func findSubCmd(root *cobra.Command, name string) *cobra.Command { - for _, c := range root.Commands() { - if c.Name() == name { - return c - } - } - return nil -} - func TestRoot_MissingConfigFileIsIgnored(t *testing.T) { viper.Reset() defer viper.Reset() diff --git a/internal/cli/update_test.go b/internal/cli/update_test.go index 6c5cc1f..e6d1748 100644 --- a/internal/cli/update_test.go +++ b/internal/cli/update_test.go @@ -327,11 +327,3 @@ func Cancelled() {} t.Fatalf("expected context canceled, got %v", err) } } - -func writeGoFileForUpdate(t *testing.T, dir, name, content string) { - t.Helper() - path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } -} diff --git a/internal/eval/metrics.go b/internal/eval/metrics.go deleted file mode 100644 index 57da209..0000000 --- a/internal/eval/metrics.go +++ /dev/null @@ -1,140 +0,0 @@ -// @index Ranking and classification metrics for parser and search evaluation. -package eval - -import "math" - -// ClassificationMetrics holds precision, recall, and F1 counts for one comparison. -// @intent expose set-based scoring fields used by parser node and edge evaluation. -type ClassificationMetrics struct { - TruePositive int - FalsePositive int - FalseNegative int - Precision float64 - Recall float64 - F1 float64 -} - -// ComputeClassification computes set-based precision, recall, and F1 for expected versus actual keys. -// @intent provide one reusable metric primitive for both parser node and edge evaluation. -func ComputeClassification(expected, actual []string) ClassificationMetrics { - expectedSet := make(map[string]bool, len(expected)) - for _, e := range expected { - expectedSet[e] = true - } - actualSet := make(map[string]bool, len(actual)) - for _, a := range actual { - actualSet[a] = true - } - - var m ClassificationMetrics - for a := range actualSet { - if expectedSet[a] { - m.TruePositive++ - } else { - m.FalsePositive++ - } - } - for e := range expectedSet { - if !actualSet[e] { - m.FalseNegative++ - } - } - - if m.TruePositive+m.FalsePositive > 0 { - m.Precision = float64(m.TruePositive) / float64(m.TruePositive+m.FalsePositive) - } - if m.TruePositive+m.FalseNegative > 0 { - m.Recall = float64(m.TruePositive) / float64(m.TruePositive+m.FalseNegative) - } - if m.Precision+m.Recall > 0 { - m.F1 = 2 * m.Precision * m.Recall / (m.Precision + m.Recall) - } - - return m -} - -// @intent measure how many of the top-k ranked results are relevant. -func PrecisionAtK(ranked []string, relevant map[string]bool, k int) float64 { - if k <= 0 || len(ranked) == 0 || len(relevant) == 0 { - return 0 - } - n := k - if n > len(ranked) { - n = len(ranked) - } - hits := 0 - for i := 0; i < n; i++ { - if relevant[ranked[i]] { - hits++ - } - } - return float64(hits) / float64(n) -} - -// @intent measure how much of the relevant set appears within the top-k ranked results. -func RecallAtK(ranked []string, relevant map[string]bool, k int) float64 { - if k <= 0 || len(ranked) == 0 || len(relevant) == 0 { - return 0 - } - n := k - if n > len(ranked) { - n = len(ranked) - } - hits := 0 - for i := 0; i < n; i++ { - if relevant[ranked[i]] { - hits++ - } - } - return float64(hits) / float64(len(relevant)) -} - -// @intent score the position of the first relevant result for ranking evaluation. -func MRR(ranked []string, relevant map[string]bool) float64 { - for i, r := range ranked { - if relevant[r] { - return 1.0 / float64(i+1) - } - } - return 0 -} - -// @intent compare ranked retrieval quality against an ideal ordering using discounted gain. -func NDCG(ranked []string, relevant map[string]bool, k int) float64 { - if k <= 0 || len(relevant) == 0 { - return 0 - } - n := k - if n > len(ranked) { - n = len(ranked) - } - - dcg := 0.0 - for i := 0; i < n; i++ { - if relevant[ranked[i]] { - dcg += 1.0 / math.Log2(float64(i+2)) - } - } - - idealCount := len(relevant) - if idealCount > k { - idealCount = k - } - idcg := 0.0 - for i := 0; i < idealCount; i++ { - idcg += 1.0 / math.Log2(float64(i+2)) - } - - if idcg == 0 { - return 0 - } - return dcg / idcg -} - -// @intent treat any returned result for a negative query as one false-positive hit in aggregate eval reporting. -func FalsePositiveRate(ranked []string) float64 { - if len(ranked) == 0 { - return 0 - } - return 1 -} diff --git a/internal/eval/metrics_test.go b/internal/eval/metrics_test.go deleted file mode 100644 index 73626d5..0000000 --- a/internal/eval/metrics_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package eval - -import ( - "math" - "testing" -) - -const epsilon = 1e-9 - -func approxEqual(a, b float64) bool { - return math.Abs(a-b) < epsilon -} - -func TestComputeClassification_AllCorrect(t *testing.T) { - expected := []string{"a", "b", "c"} - actual := []string{"a", "b", "c"} - m := ComputeClassification(expected, actual) - - if m.TruePositive != 3 { - t.Errorf("TP: got %d, want 3", m.TruePositive) - } - if m.FalsePositive != 0 { - t.Errorf("FP: got %d, want 0", m.FalsePositive) - } - if m.FalseNegative != 0 { - t.Errorf("FN: got %d, want 0", m.FalseNegative) - } - if !approxEqual(m.Precision, 1.0) { - t.Errorf("Precision: got %f, want 1.0", m.Precision) - } - if !approxEqual(m.Recall, 1.0) { - t.Errorf("Recall: got %f, want 1.0", m.Recall) - } - if !approxEqual(m.F1, 1.0) { - t.Errorf("F1: got %f, want 1.0", m.F1) - } -} - -func TestComputeClassification_PartialMatch(t *testing.T) { - expected := []string{"a", "b", "c"} - actual := []string{"a", "b", "d"} // d is FP, c is FN - m := ComputeClassification(expected, actual) - - if m.TruePositive != 2 { - t.Errorf("TP: got %d, want 2", m.TruePositive) - } - if m.FalsePositive != 1 { - t.Errorf("FP: got %d, want 1", m.FalsePositive) - } - if m.FalseNegative != 1 { - t.Errorf("FN: got %d, want 1", m.FalseNegative) - } - // Precision = 2/3 - if !approxEqual(m.Precision, 2.0/3.0) { - t.Errorf("Precision: got %f, want %f", m.Precision, 2.0/3.0) - } - // Recall = 2/3 - if !approxEqual(m.Recall, 2.0/3.0) { - t.Errorf("Recall: got %f, want %f", m.Recall, 2.0/3.0) - } -} - -func TestComputeClassification_Empty(t *testing.T) { - m := ComputeClassification(nil, nil) - if m.Precision != 0 || m.Recall != 0 || m.F1 != 0 { - t.Errorf("empty should be zeros: %+v", m) - } -} - -func TestComputeClassification_NoOverlap(t *testing.T) { - m := ComputeClassification([]string{"a", "b"}, []string{"c", "d"}) - if m.TruePositive != 0 { - t.Errorf("TP: got %d, want 0", m.TruePositive) - } - if !approxEqual(m.Precision, 0) { - t.Errorf("Precision: got %f, want 0", m.Precision) - } -} - -func TestPrecisionAtK(t *testing.T) { - relevant := map[string]bool{"a": true, "b": true, "c": true} - ranked := []string{"a", "d", "b", "e", "c"} - - if got := PrecisionAtK(ranked, relevant, 1); !approxEqual(got, 1.0) { - t.Errorf("P@1: got %f, want 1.0", got) - } - if got := PrecisionAtK(ranked, relevant, 3); !approxEqual(got, 2.0/3.0) { - t.Errorf("P@3: got %f, want %f", got, 2.0/3.0) - } - if got := PrecisionAtK(ranked, relevant, 5); !approxEqual(got, 3.0/5.0) { - t.Errorf("P@5: got %f, want %f", got, 3.0/5.0) - } -} - -func TestRecallAtK(t *testing.T) { - relevant := map[string]bool{"a": true, "b": true, "c": true} - ranked := []string{"a", "d", "b", "e", "c"} - - if got := RecallAtK(ranked, relevant, 1); !approxEqual(got, 1.0/3.0) { - t.Errorf("R@1: got %f, want %f", got, 1.0/3.0) - } - if got := RecallAtK(ranked, relevant, 5); !approxEqual(got, 1.0) { - t.Errorf("R@5: got %f, want 1.0", got) - } -} - -func TestMRR(t *testing.T) { - relevant := map[string]bool{"b": true} - ranked := []string{"a", "b", "c"} - if got := MRR(ranked, relevant); !approxEqual(got, 0.5) { - t.Errorf("MRR: got %f, want 0.5", got) - } - - // not found - if got := MRR(ranked, map[string]bool{"z": true}); !approxEqual(got, 0) { - t.Errorf("MRR not found: got %f, want 0", got) - } -} - -func TestNDCG(t *testing.T) { - relevant := map[string]bool{"a": true, "b": true, "c": true} - // Perfect ranking - perfect := []string{"a", "b", "c"} - if got := NDCG(perfect, relevant, 3); !approxEqual(got, 1.0) { - t.Errorf("nDCG perfect: got %f, want 1.0", got) - } - - // Worst: no relevant in top K - worst := []string{"x", "y", "z"} - if got := NDCG(worst, relevant, 3); !approxEqual(got, 0) { - t.Errorf("nDCG worst: got %f, want 0", got) - } -} - -func TestPrecisionAtK_EmptyInputs(t *testing.T) { - if got := PrecisionAtK(nil, nil, 5); got != 0 { - t.Errorf("empty: got %f, want 0", got) - } -} - -func TestFalsePositiveRate_EmptyRankedReturnsZero(t *testing.T) { - if got := FalsePositiveRate(nil); got != 0 { - t.Fatalf("nil ranked: got %f, want 0", got) - } - if got := FalsePositiveRate([]string{}); got != 0 { - t.Fatalf("empty ranked: got %f, want 0", got) - } -} - -func TestFalsePositiveRate_NonEmptyRankedReturnsOne(t *testing.T) { - if got := FalsePositiveRate([]string{"x"}); got != 1 { - t.Fatalf("single result: got %f, want 1", got) - } - if got := FalsePositiveRate([]string{"x", "y"}); got != 1 { - t.Fatalf("multiple results: got %f, want 1", got) - } -} diff --git a/internal/eval/parser.go b/internal/eval/parser.go deleted file mode 100644 index d840af4..0000000 --- a/internal/eval/parser.go +++ /dev/null @@ -1,315 +0,0 @@ -// @index Golden corpus loading and normalization helpers for parser evaluation. -package eval - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - - "github.com/tae2089/code-context-graph/internal/model" -) - -// @intent load every language-specific golden corpus file from the eval corpus directory tree. -func LoadGoldenDir(dir string) ([]GoldenCorpus, error) { - var corpora []GoldenCorpus - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - langDir := filepath.Join(dir, entry.Name()) - files, err := os.ReadDir(langDir) - if err != nil { - return nil, err - } - for _, f := range files { - if f.IsDir() || !strings.HasSuffix(f.Name(), ".golden.json") { - continue - } - data, err := os.ReadFile(filepath.Join(langDir, f.Name())) - if err != nil { - return nil, err - } - var c GoldenCorpus - if err := json.Unmarshal(data, &c); err != nil { - return nil, err - } - if c.Language == "" { - c.Language = entry.Name() - } - corpora = append(corpora, c) - } - } - return corpora, nil -} - -// @intent persist normalized parser output as a golden snapshot for future comparisons. -func WriteGolden(path string, corpus GoldenCorpus) error { - data, err := json.MarshalIndent(corpus, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, append(data, '\n'), 0o644) -} - -// @intent project eval nodes into stable comparison keys for set-based metrics. -func NodeKeys(nodes []EvalNode) []string { - keys := make([]string, len(nodes)) - for i, n := range nodes { - keys[i] = n.Key() - } - return keys -} - -// @intent project eval edges into stable comparison keys for set-based metrics. -func EdgeKeys(edges []EvalEdge) []string { - keys := make([]string, len(edges)) - for i, e := range edges { - keys[i] = e.Key() - } - return keys -} - -// @intent summarize parser accuracy for one language corpus by comparing expected and actual nodes and edges. -func CompareCorpus(expected, actual GoldenCorpus) LanguageReport { - nodeMetrics := ComputeClassification(NodeKeys(expected.Nodes), NodeKeys(actual.Nodes)) - edgeMetrics := ComputeClassification(EdgeKeys(expected.Edges), EdgeKeys(actual.Edges)) - return LanguageReport{ - Language: expected.Language, - NodeMetrics: nodeMetrics, - EdgeMetrics: edgeMetrics, - Files: 1, - } -} - -// @intent normalize parsed graph nodes into corpus-stable eval records independent of absolute paths. -func NormalizeNodes(nodes []model.Node, baseDir string) []EvalNode { - out := make([]EvalNode, 0, len(nodes)) - for _, n := range nodes { - if n.Kind == model.NodeKindFile { - continue - } - file := n.FilePath - if baseDir != "" { - file, _ = filepath.Rel(baseDir, file) - } - out = append(out, EvalNode{ - ID: n.QualifiedName, - Kind: string(n.Kind), - Name: n.Name, - File: file, - StartLine: n.StartLine, - EndLine: n.EndLine, - }) - } - return out -} - -// @intent normalize parsed graph edges into corpus-stable eval records keyed by qualified names. -func NormalizeEdges(edges []model.Edge, nodes []model.Node) []EvalEdge { - out := make([]EvalEdge, 0, len(edges)) - nodesByQName := make(map[string]model.Node, len(nodes)) - for _, n := range nodes { - nodesByQName[n.QualifiedName] = n - } - for _, e := range edges { - from, to := normalizeEdgeEndpoints(e, nodesByQName) - if from == "" || to == "" { - continue - } - out = append(out, EvalEdge{ - Kind: string(e.Kind), - From: from, - To: to, - }) - } - return out -} - -// normalizeEdgeEndpoints reconstructs stable edge endpoints from parser fingerprints. -// @intent eval용 edge 비교에서 파일 경로와 fingerprint를 corpus-stable 식별자로 바꾼다. -func normalizeEdgeEndpoints(edge model.Edge, nodesByQName map[string]model.Node) (string, string) { - parts := strings.Split(edge.Fingerprint, ":") - if len(parts) < 2 { - return "", "" - } - - kind := parts[0] - switch kind { - case string(model.EdgeKindContains): - if target, ok := containsTarget(edge); ok { - return edge.FilePath, target - } - case string(model.EdgeKindCalls), string(model.EdgeKindFallbackCalls), string(model.EdgeKindImportsFrom): - if len(parts) >= 4 { - if kind == string(model.EdgeKindImportsFrom) { - if target, ok := importsFromTarget(edge); ok { - return edge.FilePath, target - } - return edge.FilePath, "" - } - // fallback_calls should be compared and normalized identically to calls. - if target, ok := callsTarget(edge); ok { - from := resolveParserStageOwner(edge, nodesByQName) - return from, target - } - } - case string(model.EdgeKindTestedBy): - if len(parts) >= 4 { - from := parts[len(parts)-1] - to := strings.Join(parts[2:len(parts)-1], ":") - if from == "" || to == "" { - return "", "" - } - return from, to - } - case string(model.EdgeKindImplements): - if len(parts) >= 4 { - prefix := "implements:" + edge.FilePath + ":" - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", "" - } - rest := strings.TrimPrefix(edge.Fingerprint, prefix) - from, to, ok := strings.Cut(rest, ":") - if !ok { - return "", "" - } - if from == "" || to == "" { - return "", "" - } - return from, to - } - case string(model.EdgeKindInherits): - from, to, ok := model.ParseInheritsFingerprint(edge.FilePath, edge.Fingerprint) - if ok && from != "" && to != "" { - return from, to - } - } - return "", "" -} - -// importsFromTarget extracts the import target from an imports_from fingerprint. -// @intent import edge fingerprint를 eval 비교에 필요한 target 문자열로 복원한다. -func importsFromTarget(edge model.Edge) (string, bool) { - prefix := "imports_from:" + edge.FilePath + ":" - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", false - } - rest := strings.TrimPrefix(edge.Fingerprint, prefix) - idx := strings.LastIndex(rest, ":") - if idx < 0 { - return "", false - } - target := rest[:idx] - return target, target != "" -} - -// callsTarget extracts the callee target from a calls or fallback_calls fingerprint. -// @intent call-like edge fingerprint에서 target과 trailing line 구문이 유효한지 함께 확인한다. -func callsTarget(edge model.Edge) (string, bool) { - var prefix string - switch edge.Kind { - case model.EdgeKindCalls: - prefix = "calls:" + edge.FilePath + ":" - case model.EdgeKindFallbackCalls: - prefix = "fallback_calls:" + edge.FilePath + ":" - default: - return "", false - } - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", false - } - rest := strings.TrimPrefix(edge.Fingerprint, prefix) - idx := strings.LastIndex(rest, ":") - if idx < 0 { - return "", false - } - target := rest[:idx] - if target == "" { - return "", false - } - if _, err := parseTrailingLine(rest[idx+1:]); err != nil { - return "", false - } - return target, true -} - -// containsTarget extracts the contained symbol target from a contains fingerprint. -// @intent contains edge fingerprint를 eval 비교용 대상 심볼 이름으로 복원한다. -func containsTarget(edge model.Edge) (string, bool) { - prefix := "contains:" + edge.FilePath + ":" - if !strings.HasPrefix(edge.Fingerprint, prefix) { - return "", false - } - return strings.TrimPrefix(edge.Fingerprint, prefix), true -} - -// parseTrailingLine parses the trailing source line segment embedded in a fingerprint. -// @intent malformed fingerprint를 조기에 배제해 eval edge 비교 안정성을 높인다. -func parseTrailingLine(s string) (int, error) { - if s == "" { - return 0, os.ErrInvalid - } - line := 0 - for _, r := range s { - if r < '0' || r > '9' { - return 0, os.ErrInvalid - } - line = line*10 + int(r-'0') - } - if line <= 0 { - return 0, os.ErrInvalid - } - return line, nil -} - -// resolveParserStageOwner picks the best owner node for a parser-stage edge. -// @intent line 정보 또는 파일 내 첫 심볼을 이용해 parser fingerprint의 from endpoint를 정한다. -func resolveParserStageOwner(edge model.Edge, nodesByQName map[string]model.Node) string { - if edge.Line > 0 { - if owner := resolveOwnerByLine(edge.FilePath, edge.Line, nodesByQName); owner != "" { - return owner - } - } - for _, n := range nodesByQName { - if n.FilePath == edge.FilePath && n.Kind != model.NodeKindFile { - return n.QualifiedName - } - } - return "" -} - -// resolveOwnerByLine finds the narrowest symbol spanning a given file line. -// @intent parser-stage edge가 속한 가장 구체적인 owner 심볼을 line 범위로 찾는다. -func resolveOwnerByLine(filePath string, line int, nodesByQName map[string]model.Node) string { - var best model.Node - bestFound := false - for _, n := range nodesByQName { - if n.FilePath != filePath || n.Kind == model.NodeKindFile { - continue - } - if n.StartLine <= line && line <= n.EndLine { - if !bestFound || span(n) < span(best) || (span(n) == span(best) && n.StartLine > best.StartLine) { - best = n - bestFound = true - } - } - } - if bestFound { - return best.QualifiedName - } - return "" -} - -// span returns the inclusive line span width for a node. -// @intent owner 선택 시 더 좁은 범위의 심볼을 우선하도록 비교 지표를 제공한다. -func span(n model.Node) int { - if n.EndLine < n.StartLine { - return 0 - } - return n.EndLine - n.StartLine -} diff --git a/internal/eval/parser_test.go b/internal/eval/parser_test.go deleted file mode 100644 index ca198ff..0000000 --- a/internal/eval/parser_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package eval - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/tae2089/code-context-graph/internal/model" -) - -func TestLoadGoldenCorpus(t *testing.T) { - dir := t.TempDir() - - corpus := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - }, - Edges: []EvalEdge{ - {Kind: "calls", From: "n1", To: "n2"}, - }, - } - - data, _ := json.MarshalIndent(corpus, "", " ") - langDir := filepath.Join(dir, "go") - os.MkdirAll(langDir, 0o755) - os.WriteFile(filepath.Join(langDir, "sample.golden.json"), data, 0o644) - - loaded, err := LoadGoldenDir(dir) - if err != nil { - t.Fatalf("LoadGoldenDir: %v", err) - } - if len(loaded) != 1 { - t.Fatalf("got %d corpora, want 1", len(loaded)) - } - if loaded[0].Language != "go" { - t.Errorf("language: got %s, want go", loaded[0].Language) - } - if len(loaded[0].Nodes) != 1 { - t.Errorf("nodes: got %d, want 1", len(loaded[0].Nodes)) - } -} - -func TestNormalizeNodes(t *testing.T) { - nodes := []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - {ID: "n2", Kind: "class", Name: "World", File: "sample.go", StartLine: 10}, - } - keys := NodeKeys(nodes) - if len(keys) != 2 { - t.Fatalf("got %d keys, want 2", len(keys)) - } - if keys[0] != "function:Hello@sample.go" { - t.Errorf("key[0]: got %s", keys[0]) - } -} - -func TestCompareCorpus(t *testing.T) { - expected := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - {ID: "n2", Kind: "function", Name: "World", File: "sample.go", StartLine: 10}, - }, - Edges: []EvalEdge{ - {Kind: "calls", From: "n1", To: "n2"}, - }, - } - - actual := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - {ID: "n3", Kind: "function", Name: "Extra", File: "sample.go", StartLine: 20}, - }, - Edges: []EvalEdge{ - {Kind: "calls", From: "n1", To: "n2"}, - }, - } - - report := CompareCorpus(expected, actual) - if report.NodeMetrics.TruePositive != 1 { - t.Errorf("node TP: got %d, want 1", report.NodeMetrics.TruePositive) - } - if report.NodeMetrics.FalsePositive != 1 { - t.Errorf("node FP: got %d, want 1", report.NodeMetrics.FalsePositive) - } - if report.NodeMetrics.FalseNegative != 1 { - t.Errorf("node FN: got %d, want 1", report.NodeMetrics.FalseNegative) - } - if report.EdgeMetrics.TruePositive != 1 { - t.Errorf("edge TP: got %d, want 1", report.EdgeMetrics.TruePositive) - } -} - -func TestUpdateGolden(t *testing.T) { - dir := t.TempDir() - langDir := filepath.Join(dir, "go") - os.MkdirAll(langDir, 0o755) - - corpus := GoldenCorpus{ - Language: "go", - File: "sample.go", - Nodes: []EvalNode{ - {ID: "n1", Kind: "function", Name: "Hello", File: "sample.go", StartLine: 3}, - }, - } - - outPath := filepath.Join(langDir, "sample.golden.json") - err := WriteGolden(outPath, corpus) - if err != nil { - t.Fatalf("WriteGolden: %v", err) - } - - loaded, err := LoadGoldenDir(dir) - if err != nil { - t.Fatalf("reload: %v", err) - } - if len(loaded) != 1 || loaded[0].Nodes[0].Name != "Hello" { - t.Errorf("round-trip failed: %+v", loaded) - } -} - -func TestNormalizeEdges_PreservesParserStageEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "sample.Hello", Kind: model.NodeKindFunction, Name: "Hello", FilePath: "sample.go", StartLine: 3, EndLine: 5}, - {ID: 0, QualifiedName: "sample.World", Kind: model.NodeKindFunction, Name: "World", FilePath: "sample.go", StartLine: 10, EndLine: 12}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindCalls, FilePath: "sample.go", Line: 4, Fingerprint: "calls:sample.go:sample.World:4"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "sample.Hello" { - t.Fatalf("from collapsed: got %q, want %q", actual[0].From, "sample.Hello") - } - if actual[0].To != "sample.World" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "sample.World") - } -} - -func TestNormalizeEdges_PreservesFallbackCallEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "sample.Hello", Kind: model.NodeKindFunction, Name: "Hello", FilePath: "sample.go", StartLine: 3, EndLine: 5}, - {ID: 0, QualifiedName: "sample.World", Kind: model.NodeKindFunction, Name: "World", FilePath: "sample.go", StartLine: 10, EndLine: 12}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindFallbackCalls, FilePath: "sample.go", Line: 4, Fingerprint: "fallback_calls:sample.go:sample.World:4"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "sample.Hello" { - t.Fatalf("from collapsed: got %q, want %q", actual[0].From, "sample.Hello") - } - if actual[0].To != "sample.World" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "sample.World") - } -} - -func TestNormalizeEdges_ImportsFromUsesFilePathAndFullTarget(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.File", Kind: model.NodeKindFile, Name: "File", FilePath: "dir:with:colon/sample.go"}, - {ID: 0, QualifiedName: "pkg.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "dir:with:colon/sample.go", StartLine: 3, EndLine: 5}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindImportsFrom, FilePath: "dir:with:colon/sample.go", Line: 4, Fingerprint: "imports_from:dir:with:colon/sample.go:github.com/acme/lib:v2/util:4"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "dir:with:colon/sample.go" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "dir:with:colon/sample.go") - } - if actual[0].To != "github.com/acme/lib:v2/util" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util") - } -} - -func TestNormalizeEdges_TestedByUsesTestQNameAsFrom(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "sample.TestHello", Kind: model.NodeKindFunction, Name: "TestHello", FilePath: "sample_test.go", StartLine: 3, EndLine: 5}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "sample.go", StartLine: 10, EndLine: 12}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindTestedBy, FilePath: "sample_test.go", Line: 4, Fingerprint: "tested_by:sample_test.go:github.com/acme/lib:v2/util.Helper:sample.TestHello"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "sample.TestHello" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "sample.TestHello") - } - if actual[0].To != "github.com/acme/lib:v2/util.Helper" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Helper") - } -} - -func TestNormalizeEdges_ImplementsUsesFingerprintEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Interface", Kind: model.NodeKindClass, Name: "Interface", FilePath: "iface.go", StartLine: 10, EndLine: 12}, - {ID: 0, QualifiedName: "pkg.Implementation", Kind: model.NodeKindClass, Name: "Implementation", FilePath: "impl.go", StartLine: 20, EndLine: 24}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindImplements, FilePath: "parser.go", Line: 0, Fingerprint: "implements:parser.go:pkg.Implementation:pkg.Interface"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "pkg.Implementation" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "pkg.Implementation") - } - if actual[0].To != "pkg.Interface" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "pkg.Interface") - } -} - -func TestNormalizeEdges_InheritsUsesFingerprintEndpoints(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Child", Kind: model.NodeKindClass, Name: "Child", FilePath: "models.py", StartLine: 10, EndLine: 20}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Base", Kind: model.NodeKindClass, Name: "Base", FilePath: "models.py", StartLine: 30, EndLine: 40}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindInherits, FilePath: "models.py", Line: 0, Fingerprint: model.BuildInheritsFingerprintV2("models.py", "pkg.Child", "github.com/acme/lib:v2/util.Base")}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "pkg.Child" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "pkg.Child") - } - if actual[0].To != "github.com/acme/lib:v2/util.Base" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Base") - } -} - -func TestNormalizeEdges_CallsUsesLastColonSafeTargetAndNumericLine(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Owner", Kind: model.NodeKindFunction, Name: "Owner", FilePath: "dir:with:colon/parser.go", StartLine: 10, EndLine: 20}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "dir:with:colon/parser.go", StartLine: 30, EndLine: 40}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindCalls, FilePath: "dir:with:colon/parser.go", Line: 12, Fingerprint: "calls:dir:with:colon/parser.go:github.com/acme/lib:v2/util.Helper:12"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "pkg.Owner" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "pkg.Owner") - } - if actual[0].To != "github.com/acme/lib:v2/util.Helper" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Helper") - } -} - -func TestNormalizeEdges_ContainsUsesFullTargetAfterFilePathPrefix(t *testing.T) { - nodes := []model.Node{ - {ID: 0, QualifiedName: "pkg.Owner", Kind: model.NodeKindFunction, Name: "Owner", FilePath: "dir:with:colon/parser.go", StartLine: 10, EndLine: 20}, - {ID: 0, QualifiedName: "github.com/acme/lib:v2/util.Helper", Kind: model.NodeKindFunction, Name: "Helper", FilePath: "dir:with:colon/parser.go", StartLine: 30, EndLine: 40}, - } - edges := []model.Edge{ - {Kind: model.EdgeKindContains, FilePath: "dir:with:colon/parser.go", Line: 11, Fingerprint: "contains:dir:with:colon/parser.go:github.com/acme/lib:v2/util.Helper"}, - } - - actual := NormalizeEdges(edges, nodes) - if len(actual) != 1 { - t.Fatalf("got %d edges, want 1", len(actual)) - } - if actual[0].From != "dir:with:colon/parser.go" { - t.Fatalf("from mismatch: got %q, want %q", actual[0].From, "dir:with:colon/parser.go") - } - if actual[0].To != "github.com/acme/lib:v2/util.Helper" { - t.Fatalf("to mismatch: got %q, want %q", actual[0].To, "github.com/acme/lib:v2/util.Helper") - } -} diff --git a/internal/eval/runner.go b/internal/eval/runner.go deleted file mode 100644 index 8984c1a..0000000 --- a/internal/eval/runner.go +++ /dev/null @@ -1,232 +0,0 @@ -// @index Eval runner orchestration for parser and search benchmark suites. -package eval - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/tae2089/code-context-graph/internal/parse/treesitter" -) - -// RunOptions collects corpus paths, parser walkers, and output settings for one eval invocation. -// @intent configure parser/search evaluation suites from CLI without leaking command details. -type RunOptions struct { - CorpusDir string - Suite string - Format string - Update bool - Walkers map[string]*treesitter.Walker - SearchFn SearchFunc - Writer io.Writer -} - -// Run executes the requested evaluation suites and writes the chosen report format. -// @intent provide one orchestration entry point for CLI-driven parser and search evaluation. -func Run(ctx context.Context, opts RunOptions) (*Report, error) { - report := &Report{Suite: opts.Suite} - - if opts.Suite == "all" || opts.Suite == "parser" { - if err := runParserEval(ctx, opts, report); err != nil { - return nil, fmt.Errorf("parser eval: %w", err) - } - } - - if opts.Suite == "all" || opts.Suite == "search" { - if err := runSearchEval(ctx, opts, report); err != nil { - return nil, fmt.Errorf("search eval: %w", err) - } - } - - if opts.Format == "json" { - return report, writeJSON(opts.Writer, report) - } - return report, writeTable(opts.Writer, report) -} - -// @intent route parser evaluation into update or comparison mode without duplicating top-level flow control. -func runParserEval(ctx context.Context, opts RunOptions, report *Report) error { - if opts.Update { - return runParserUpdate(ctx, opts) - } - return runParserCompare(ctx, opts, report) -} - -// @intent regenerate golden parser snapshots from the current parser implementation. -func runParserUpdate(ctx context.Context, opts RunOptions) error { - entries, err := os.ReadDir(opts.CorpusDir) - if err != nil { - return err - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - lang := entry.Name() - walker, ok := opts.Walkers[lang] - if !ok { - continue - } - langDir := filepath.Join(opts.CorpusDir, lang) - files, err := os.ReadDir(langDir) - if err != nil { - return err - } - for _, f := range files { - if f.IsDir() || strings.HasSuffix(f.Name(), ".golden.json") { - continue - } - srcFile := filepath.Join(langDir, f.Name()) - src, err := readFileContent(srcFile) - if err != nil { - return fmt.Errorf("read source %s: %w", srcFile, err) - } - nodes, edges, _, err := walker.ParseWithComments(ctx, srcFile, src) - if err != nil { - return fmt.Errorf("parse %s: %w", srcFile, err) - } - actualNodes := NormalizeNodes(nodes, langDir) - actualEdges := NormalizeEdges(edges, nodes) - - golden := GoldenCorpus{ - Language: lang, - File: f.Name(), - Nodes: actualNodes, - Edges: actualEdges, - } - goldenPath := filepath.Join(langDir, f.Name()+".golden.json") - if err := WriteGolden(goldenPath, golden); err != nil { - return fmt.Errorf("write golden %s: %w", goldenPath, err) - } - } - } - return nil -} - -// @intent compare current parser output against stored golden corpora and accumulate language reports. -func runParserCompare(ctx context.Context, opts RunOptions, report *Report) error { - corpora, err := LoadGoldenDir(opts.CorpusDir) - if err != nil { - return err - } - - byLang := make(map[string][]GoldenCorpus) - for _, c := range corpora { - byLang[c.Language] = append(byLang[c.Language], c) - } - - for lang, goldens := range byLang { - walker, ok := opts.Walkers[lang] - if !ok { - continue - } - - var allExpectedNodes, allActualNodes []string - var allExpectedEdges, allActualEdges []string - fileCount := 0 - - for _, golden := range goldens { - langDir := filepath.Join(opts.CorpusDir, lang) - srcFile := filepath.Join(langDir, golden.File) - src, err := readFileContent(srcFile) - if err != nil { - return fmt.Errorf("read source %s: %w", srcFile, err) - } - - nodes, edges, _, err := walker.ParseWithComments(ctx, srcFile, src) - if err != nil { - return fmt.Errorf("parse %s: %w", srcFile, err) - } - - actualNodes := NormalizeNodes(nodes, langDir) - actualEdges := NormalizeEdges(edges, nodes) - - allExpectedNodes = append(allExpectedNodes, NodeKeys(golden.Nodes)...) - allActualNodes = append(allActualNodes, NodeKeys(actualNodes)...) - allExpectedEdges = append(allExpectedEdges, EdgeKeys(golden.Edges)...) - allActualEdges = append(allActualEdges, EdgeKeys(actualEdges)...) - fileCount++ - } - - report.Languages = append(report.Languages, LanguageReport{ - Language: lang, - NodeMetrics: ComputeClassification(allExpectedNodes, allActualNodes), - EdgeMetrics: ComputeClassification(allExpectedEdges, allActualEdges), - Files: fileCount, - }) - } - - return nil -} - -// @intent run ranking metrics over the search corpus when a search backend is available. -func runSearchEval(_ context.Context, opts RunOptions, report *Report) error { - queryPath := fmt.Sprintf("%s/queries.json", opts.CorpusDir) - qc, err := LoadQueryCorpus(queryPath) - if err != nil { - return nil - } - - if opts.SearchFn == nil { - return fmt.Errorf("search function not configured") - } - - sr, err := EvaluateQueries(qc.Queries, opts.SearchFn) - if err != nil { - return err - } - report.Search = &sr - return nil -} - -// @intent centralize file reads used by parser evaluation so tests can characterize failures consistently. -func readFileContent(path string) ([]byte, error) { - return os.ReadFile(path) -} - -// @intent emit the eval report as pretty-printed JSON for machine consumption. -func writeJSON(w io.Writer, report *Report) error { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(report) -} - -// @intent render a compact human-readable summary of parser and search evaluation metrics. -func writeTable(w io.Writer, report *Report) error { - if len(report.Languages) > 0 { - fmt.Fprintf(w, "=== Parser Evaluation ===\n\n") - fmt.Fprintf(w, "%-14s %6s %8s %8s %8s %8s %8s %8s\n", - "Language", "Files", "Node P", "Node R", "Node F1", "Edge P", "Edge R", "Edge F1") - fmt.Fprintf(w, "%s\n", strings.Repeat("─", 82)) - - for _, lr := range report.Languages { - fmt.Fprintf(w, "%-14s %6d %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f\n", - lr.Language, lr.Files, - lr.NodeMetrics.Precision, lr.NodeMetrics.Recall, lr.NodeMetrics.F1, - lr.EdgeMetrics.Precision, lr.EdgeMetrics.Recall, lr.EdgeMetrics.F1) - } - fmt.Fprintln(w) - } - - if report.Search != nil { - fmt.Fprintf(w, "=== Search Evaluation ===\n\n") - fmt.Fprintf(w, "Queries: %d\n", report.Search.QueriesTotal) - fmt.Fprintf(w, "P@1: %.4f\n", report.Search.AvgPAt1) - fmt.Fprintf(w, "P@3: %.4f\n", report.Search.AvgPAt3) - fmt.Fprintf(w, "P@5: %.4f\n", report.Search.AvgPAt5) - fmt.Fprintf(w, "R@5: %.4f\n", report.Search.AvgRecallAt5) - fmt.Fprintf(w, "MRR: %.4f\n", report.Search.AvgMRR) - fmt.Fprintf(w, "nDCG@5: %.4f\n", report.Search.AvgNDCGAt5) - if report.Search.NegativeQueries > 0 { - fmt.Fprintf(w, "Negatives: %d\n", report.Search.NegativeQueries) - fmt.Fprintf(w, "FP: %d\n", report.Search.NegativeFalsePositives) - fmt.Fprintf(w, "Pass Rate: %.4f\n", report.Search.NegativePassRate) - } - } - - return nil -} diff --git a/internal/eval/runner_test.go b/internal/eval/runner_test.go deleted file mode 100644 index 821880e..0000000 --- a/internal/eval/runner_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package eval - -import ( - "bytes" - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tae2089/code-context-graph/internal/parse/treesitter" -) - -func TestRunParserEval_Update(t *testing.T) { - walker := treesitter.NewWalker(treesitter.GoSpec) - - dir := t.TempDir() - copyTestCorpus(t, dir, "go", "sample.go", `package sample - -func Hello() {} - -func World() { - Hello() -} -`) - - var buf bytes.Buffer - opts := RunOptions{ - CorpusDir: dir, - Suite: "parser", - Format: "table", - Update: true, - Walkers: map[string]*treesitter.Walker{"go": walker}, - Writer: &buf, - } - - _, err := Run(context.Background(), opts) - if err != nil { - t.Fatalf("Run update: %v", err) - } - - corpora, err := LoadGoldenDir(dir) - if err != nil { - t.Fatalf("LoadGoldenDir: %v", err) - } - if len(corpora) != 1 { - t.Fatalf("corpora: got %d, want 1", len(corpora)) - } - - found := false - for _, n := range corpora[0].Nodes { - if n.Name == "Hello" && n.Kind == "function" { - found = true - } - } - if !found { - t.Errorf("expected Hello function in golden, got: %+v", corpora[0].Nodes) - } -} - -func TestRunParserEval_Compare(t *testing.T) { - walker := treesitter.NewWalker(treesitter.GoSpec) - - dir := t.TempDir() - copyTestCorpus(t, dir, "go", "sample.go", `package sample - -func Hello() {} - -func World() { - Hello() -} -`) - - walkers := map[string]*treesitter.Walker{"go": walker} - - var buf bytes.Buffer - _, err := Run(context.Background(), RunOptions{ - CorpusDir: dir, Suite: "parser", Format: "table", - Update: true, Walkers: walkers, Writer: &buf, - }) - if err != nil { - t.Fatalf("update: %v", err) - } - - buf.Reset() - report, err := Run(context.Background(), RunOptions{ - CorpusDir: dir, Suite: "parser", Format: "table", - Walkers: walkers, Writer: &buf, - }) - if err != nil { - t.Fatalf("compare: %v", err) - } - - if len(report.Languages) != 1 { - t.Fatalf("languages: got %d, want 1", len(report.Languages)) - } - - lr := report.Languages[0] - if !approxEqual(lr.NodeMetrics.F1, 1.0) { - t.Errorf("node F1: got %f, want 1.0", lr.NodeMetrics.F1) - } - if buf.Len() == 0 { - t.Error("expected table output") - } -} - -func copyTestCorpus(t *testing.T, dir, lang, filename, content string) { - t.Helper() - langDir := filepath.Join(dir, lang) - os.MkdirAll(langDir, 0o755) - os.WriteFile(filepath.Join(langDir, filename), []byte(content), 0o644) -} - -func TestWriteTable_RendersNegativeControlBlock(t *testing.T) { - var buf bytes.Buffer - report := &Report{Search: &SearchReport{ - QueriesTotal: 2, - AvgPAt1: 1.0, - AvgPAt3: 0.8333, - AvgPAt5: 0.8, - AvgRecallAt5: 1.0, - AvgMRR: 1.0, - AvgNDCGAt5: 1.0, - NegativeQueries: 1, - NegativeFalsePositives: 0, - NegativePassRate: 1.0, - }} - - if err := writeTable(&buf, report); err != nil { - t.Fatalf("writeTable: %v", err) - } - out := buf.String() - for _, want := range []string{"Negatives:", "FP:", "Pass Rate: 1.0000"} { - if !strings.Contains(out, want) { - t.Fatalf("expected output to contain %q, got:\n%s", want, out) - } - } -} - -func TestWriteTable_OmitsNegativeBlock_WhenZeroNegatives(t *testing.T) { - var buf bytes.Buffer - report := &Report{Search: &SearchReport{ - QueriesTotal: 1, - AvgPAt1: 1.0, - AvgPAt3: 1.0, - AvgPAt5: 1.0, - AvgRecallAt5: 1.0, - AvgMRR: 1.0, - AvgNDCGAt5: 1.0, - NegativeQueries: 0, - }} - - if err := writeTable(&buf, report); err != nil { - t.Fatalf("writeTable: %v", err) - } - if strings.Contains(buf.String(), "Negatives:") { - t.Fatalf("did not expect negative block, got:\n%s", buf.String()) - } -} diff --git a/internal/eval/schema.go b/internal/eval/schema.go deleted file mode 100644 index 551f04f..0000000 --- a/internal/eval/schema.go +++ /dev/null @@ -1,105 +0,0 @@ -// @index Data schemas for parser golden corpora and search evaluation reports. -package eval - -// EvalNode is the normalized parser node shape used in golden corpora. -// @intent capture corpus-stable node identity independent of absolute paths. -type EvalNode struct { - ID string `json:"id"` - Kind string `json:"kind"` - Name string `json:"name"` - File string `json:"file"` - StartLine int `json:"start_line"` - EndLine int `json:"end_line,omitempty"` -} - -// Key derives a stable string key for node-level evaluation comparisons. -// @intent provide a deterministic identifier for set-based parser metrics. -func (n EvalNode) Key() string { - return n.Kind + ":" + n.Name + "@" + n.File -} - -// EvalEdge is the normalized parser edge shape used in golden corpora. -// @intent capture corpus-stable edge identity for parser comparisons. -type EvalEdge struct { - Kind string `json:"kind"` - From string `json:"from"` - To string `json:"to"` -} - -// Key derives a stable string key for edge-level evaluation comparisons. -// @intent provide a deterministic identifier for set-based parser metrics. -func (e EvalEdge) Key() string { - return e.Kind + ":" + e.From + "->" + e.To -} - -// GoldenCorpus stores one parser snapshot for a source file in a language corpus. -// @intent persist expected parser output for regression comparison. -type GoldenCorpus struct { - Language string `json:"language"` - File string `json:"file"` - Nodes []EvalNode `json:"nodes"` - Edges []EvalEdge `json:"edges"` -} - -// QueryCase defines one search evaluation query and its relevant expected results. -// @intent describe a single ranked-retrieval test case for search evaluation. -type QueryCase struct { - Query string `json:"query"` - Relevant []string `json:"relevant"` - K int `json:"k,omitempty"` -} - -// QueryCorpus groups search evaluation cases loaded from one corpus directory. -// @intent represent the full search evaluation suite as one loadable artifact. -type QueryCorpus struct { - CorpusDir string `json:"corpus_dir"` - Queries []QueryCase `json:"queries"` -} - -// LanguageReport summarizes parser accuracy metrics for one language corpus. -// @intent expose per-language node and edge metrics in the eval report. -type LanguageReport struct { - Language string `json:"language"` - NodeMetrics ClassificationMetrics `json:"node_metrics"` - EdgeMetrics ClassificationMetrics `json:"edge_metrics"` - Files int `json:"files"` -} - -// SearchReport holds aggregate ranking metrics across the search evaluation corpus. -// @intent surface average P@K, recall, MRR, and nDCG over all search queries. -type SearchReport struct { - QueriesTotal int `json:"queries_total"` - AvgPAt1 float64 `json:"avg_p_at_1"` - AvgPAt3 float64 `json:"avg_p_at_3"` - AvgPAt5 float64 `json:"avg_p_at_5"` - AvgRecallAt5 float64 `json:"avg_recall_at_5"` - AvgMRR float64 `json:"avg_mrr"` - AvgNDCGAt5 float64 `json:"avg_ndcg_at_5"` - NegativeQueries int `json:"negative_queries"` - NegativeFalsePositives int `json:"negative_false_positives"` - NegativePassRate float64 `json:"negative_pass_rate"` - PerQuery []QueryDiagnostic `json:"per_query,omitempty"` -} - -// QueryDiagnostic records one query's detailed evaluation outcome for debugging regressions. -// @intent aggregate metric만으로 보이지 않는 개별 쿼리 실패 원인을 보존한다. -type QueryDiagnostic struct { - Query string `json:"query"` - Kind string `json:"kind"` - ResultsReturned int `json:"results_returned"` - PAt1 float64 `json:"p_at_1,omitempty"` - PAt5 float64 `json:"p_at_5,omitempty"` - RecallAt5 float64 `json:"recall_at_5,omitempty"` - MRR float64 `json:"mrr,omitempty"` - NDCGAt5 float64 `json:"ndcg_at_5,omitempty"` - FalsePositive bool `json:"false_positive,omitempty"` - TopResults []string `json:"top_results,omitempty"` -} - -// Report bundles parser and search evaluation output into one CLI-facing payload. -// @intent represent the complete eval result returned to CLI and JSON consumers. -type Report struct { - Suite string `json:"suite"` - Languages []LanguageReport `json:"languages,omitempty"` - Search *SearchReport `json:"search,omitempty"` -} diff --git a/internal/eval/search.go b/internal/eval/search.go deleted file mode 100644 index 9241499..0000000 --- a/internal/eval/search.go +++ /dev/null @@ -1,124 +0,0 @@ -// @index Search evaluation corpus loading and ranking metric orchestration. -package eval - -import ( - "encoding/json" - "os" -) - -// capResults truncates a result slice to at most n items. -// @intent per-query 진단에서 상위 결과만 제한해 출력 부피를 제어한다. -func capResults(results []string, n int) []string { - if len(results) <= n { - return results - } - return results[:n] -} - -// SearchFunc abstracts search execution so evaluation can run against any backend. -// @intent decouple ranking metrics from concrete search implementations. -type SearchFunc func(query string, limit int) ([]string, error) - -// LoadQueryCorpus loads search evaluation cases from a JSON corpus file. -// @intent ingest the search query corpus before running ranking evaluation. -func LoadQueryCorpus(path string) (QueryCorpus, error) { - data, err := os.ReadFile(path) - if err != nil { - return QueryCorpus{}, err - } - var qc QueryCorpus - if err := json.Unmarshal(data, &qc); err != nil { - return QueryCorpus{}, err - } - return qc, nil -} - -// @intent execute ranked retrieval metrics across all search evaluation cases. -func EvaluateQueries(cases []QueryCase, searchFn SearchFunc) (SearchReport, error) { - if len(cases) == 0 { - return SearchReport{}, nil - } - - report := SearchReport{QueriesTotal: len(cases)} - var sumP1, sumP3, sumP5, sumR5, sumMRR, sumNDCG5 float64 - negativeQueries := 0 - negativeFalsePositives := 0 - positiveQueries := 0 - - for _, qc := range cases { - k := qc.K - if k <= 0 { - k = 5 - } - limit := k - if limit < 5 { - limit = 5 - } - - results, err := searchFn(qc.Query, limit) - if err != nil { - return SearchReport{}, err - } - - if len(qc.Relevant) == 0 { - negativeQueries++ - falsePositive := FalsePositiveRate(results) > 0 - negativeFalsePositives += int(FalsePositiveRate(results)) - report.PerQuery = append(report.PerQuery, QueryDiagnostic{ - Query: qc.Query, - Kind: "negative", - ResultsReturned: len(results), - FalsePositive: falsePositive, - TopResults: capResults(results, 5), - }) - continue - } - positiveQueries++ - - relevant := make(map[string]bool, len(qc.Relevant)) - for _, r := range qc.Relevant { - relevant[r] = true - } - - p1 := PrecisionAtK(results, relevant, 1) - p3 := PrecisionAtK(results, relevant, 3) - p5 := PrecisionAtK(results, relevant, 5) - r5 := RecallAtK(results, relevant, 5) - mrr := MRR(results, relevant) - ndcg5 := NDCG(results, relevant, 5) - - sumP1 += p1 - sumP3 += p3 - sumP5 += p5 - sumR5 += r5 - sumMRR += mrr - sumNDCG5 += ndcg5 - report.PerQuery = append(report.PerQuery, QueryDiagnostic{ - Query: qc.Query, - Kind: "positive", - ResultsReturned: len(results), - PAt1: p1, - PAt5: p5, - RecallAt5: r5, - MRR: mrr, - NDCGAt5: ndcg5, - TopResults: capResults(results, 5), - }) - } - - report.NegativeQueries = negativeQueries - report.NegativeFalsePositives = negativeFalsePositives - if positiveQueries > 0 { - n := float64(positiveQueries) - report.AvgPAt1 = sumP1 / n - report.AvgPAt3 = sumP3 / n - report.AvgPAt5 = sumP5 / n - report.AvgRecallAt5 = sumR5 / n - report.AvgMRR = sumMRR / n - report.AvgNDCGAt5 = sumNDCG5 / n - } - if negativeQueries > 0 { - report.NegativePassRate = 1 - float64(negativeFalsePositives)/float64(negativeQueries) - } - return report, nil -} diff --git a/internal/eval/search_test.go b/internal/eval/search_test.go deleted file mode 100644 index f62a0f6..0000000 --- a/internal/eval/search_test.go +++ /dev/null @@ -1,451 +0,0 @@ -package eval - -import ( - "encoding/json" - "os" - "path/filepath" - "runtime" - "slices" - "testing" -) - -func repoRoot(t *testing.T) string { - t.Helper() - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) -} - -func TestLoadQueryCorpus(t *testing.T) { - dir := t.TempDir() - qc := QueryCorpus{ - CorpusDir: "testdata/eval/go", - Queries: []QueryCase{ - {Query: "Hello", Relevant: []string{"function:Hello@sample.go"}, K: 5}, - {Query: "auth", Relevant: []string{"function:Authenticate@auth.go"}}, - }, - } - data, _ := json.MarshalIndent(qc, "", " ") - os.WriteFile(dir+"/queries.json", data, 0o644) - - loaded, err := LoadQueryCorpus(dir + "/queries.json") - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 2 { - t.Fatalf("queries: got %d, want 2", len(loaded.Queries)) - } - if loaded.Queries[0].K != 5 { - t.Errorf("K: got %d, want 5", loaded.Queries[0].K) - } -} - -func TestLoadQueryCorpus_IncludesJavaCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - if loaded.Queries[5].Relevant[0] != "class:UserService@Sample.java" { - t.Fatalf("relevant: got %q, want %q", loaded.Queries[5].Relevant[0], "class:UserService@Sample.java") - } -} - -func TestLoadQueryCorpus_IncludesRustCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - if loaded.Queries[6].Relevant[0] != "function:get_user@sample.rs" { - t.Fatalf("relevant: got %q, want %q", loaded.Queries[6].Relevant[0], "function:get_user@sample.rs") - } -} - -func TestLoadQueryCorpus_IncludesJavaScriptCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - if loaded.Queries[7].Relevant[0] != "function:getUser@sample.js" { - t.Fatalf("relevant: got %q, want %q", loaded.Queries[7].Relevant[0], "function:getUser@sample.js") - } -} - -func TestLoadQueryCorpus_IncludesKotlinCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - for _, q := range loaded.Queries { - if hasRelevant(q.Relevant, "function:getUser@Sample.kt") { - return - } - } - t.Fatal("missing Kotlin relevant ID function:getUser@Sample.kt") -} - -func TestLoadQueryCorpus_IncludesPHPCase(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - for _, q := range loaded.Queries { - if hasRelevant(q.Relevant, "function:getUser@sample.php") { - return - } - } - t.Fatal("missing PHP relevant ID function:getUser@sample.php") -} - -func TestLoadQueryCorpus_CoversRemainingLanguages(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - if len(loaded.Queries) != 15 { - t.Fatalf("queries: got %d, want 15", len(loaded.Queries)) - } - - wants := []string{ - "function:get_user@sample.rb", - "function:get_user@sample.c", - "function:getUser@sample.cpp", - "function:UserService:getUser@sample.lua", - } - for _, want := range wants { - found := false - for _, q := range loaded.Queries { - if hasRelevant(q.Relevant, want) { - found = true - break - } - } - if !found { - t.Fatalf("missing relevant ID %s", want) - } - } -} - -func TestLoadQueryCorpus_UsesFixtureAlignedQueryTexts(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - - var python, typescript, rust, javascript bool - for _, q := range loaded.Queries { - switch q.Query { - case "get_user": - python = hasRelevant(q.Relevant, "function:get_user@sample.py") - case "getUser": - typescript = hasRelevant(q.Relevant, "function:getUser@sample.ts") - case "get_user Rust": - rust = hasRelevant(q.Relevant, "function:get_user@sample.rs") - case "getUser JavaScript": - javascript = hasRelevant(q.Relevant, "function:getUser@sample.js") - } - } - - if !python { - t.Fatal("missing aligned Python query case") - } - if !typescript { - t.Fatal("missing aligned TypeScript query case") - } - if !rust { - t.Fatal("missing aligned Rust query case") - } - if !javascript { - t.Fatal("missing aligned JavaScript query case") - } -} - -func TestLoadQueryCorpus_BareNameQueriesAreMultiRelevant(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - - expectations := map[string][]string{ - "UserService": { - "class:UserService@sample.go", - "class:UserService@Sample.java", - "class:UserService@sample.php", - "class:UserService@sample.cpp", - }, - "get_user": { - "function:get_user@sample.py", - "function:get_user@sample.rs", - "function:get_user@sample.rb", - "function:get_user@sample.c", - }, - "getUser": { - "function:getUser@sample.ts", - "function:getUser@sample.js", - "function:getUser@Sample.kt", - "function:getUser@sample.php", - "function:getUser@sample.cpp", - }, - } - - for query, wants := range expectations { - var got []string - for _, q := range loaded.Queries { - if q.Query == query { - got = q.Relevant - break - } - } - for _, want := range wants { - if !hasRelevant(got, want) { - t.Fatalf("query %q missing relevant ID %q (got %v)", query, want, got) - } - } - } -} - -func TestLoadQueryCorpus_IncludesNegativeControl(t *testing.T) { - loaded, err := LoadQueryCorpus(filepath.Join(repoRoot(t), "testdata", "eval", "queries.json")) - if err != nil { - t.Fatalf("LoadQueryCorpus: %v", err) - } - for _, q := range loaded.Queries { - if q.Query != "" && len(q.Relevant) == 0 { - return - } - } - t.Fatal("missing negative-control query with empty relevant set") -} - -func hasRelevant(relevant []string, want string) bool { - return slices.Contains(relevant, want) -} - -func TestEvaluateQueries(t *testing.T) { - cases := []QueryCase{ - { - Query: "Hello", - Relevant: []string{"a", "b"}, - K: 3, - }, - } - - mockSearch := func(query string, limit int) ([]string, error) { - return []string{"a", "c", "b"}, nil - } - - report, err := EvaluateQueries(cases, mockSearch) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if report.QueriesTotal != 1 { - t.Errorf("total: got %d, want 1", report.QueriesTotal) - } - if !approxEqual(report.AvgPAt1, 1.0) { - t.Errorf("P@1: got %f, want 1.0", report.AvgPAt1) - } - if !approxEqual(report.AvgMRR, 1.0) { - t.Errorf("MRR: got %f, want 1.0", report.AvgMRR) - } - if !approxEqual(report.AvgRecallAt5, 1.0) { - t.Errorf("R@5: got %f, want 1.0 (all relevant found within K=3)", report.AvgRecallAt5) - } -} - -func TestEvaluateQueries_Empty(t *testing.T) { - report, err := EvaluateQueries(nil, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if report.QueriesTotal != 0 { - t.Errorf("total: got %d, want 0", report.QueriesTotal) - } -} - -func TestEvaluateQueries_NegativeControlExcludedFromRankingAverages(t *testing.T) { - cases := []QueryCase{ - {Query: "positive", Relevant: []string{"a"}, K: 5}, - {Query: "negative", Relevant: []string{}, K: 5}, - } - - searchFn := func(query string, limit int) ([]string, error) { - if query == "positive" { - return []string{"a"}, nil - } - return nil, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) { - t.Fatalf("positive averages should remain perfect, got P@1=%f MRR=%f R@5=%f", report.AvgPAt1, report.AvgMRR, report.AvgRecallAt5) - } - if report.NegativeQueries != 1 || report.NegativeFalsePositives != 0 || !approxEqual(report.NegativePassRate, 1.0) { - t.Fatalf("negative control aggregation mismatch: %+v", report) - } - if report.QueriesTotal != 2 { - t.Fatalf("total queries: got %d, want 2", report.QueriesTotal) - } -} - -func TestEvaluateQueries_NegativeControlDetectsLeak(t *testing.T) { - cases := []QueryCase{{Query: "negative", Relevant: []string{}, K: 5}} - searchFn := func(query string, limit int) ([]string, error) { - return []string{"unexpected"}, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if report.NegativeQueries != 1 || report.NegativeFalsePositives != 1 || !approxEqual(report.NegativePassRate, 0.0) { - t.Fatalf("negative leak aggregation mismatch: %+v", report) - } -} - -func TestEvaluateQueries_PositiveBaselineUnchanged(t *testing.T) { - cases := []QueryCase{{Query: "Hello", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(query string, limit int) ([]string, error) { - return []string{"a", "c", "b"}, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) { - t.Fatalf("positive baseline changed: %+v", report) - } -} - -func TestEvaluateQueries_EmitsPerQueryDiagnostics(t *testing.T) { - cases := []QueryCase{ - {Query: "hit", Relevant: []string{"a", "b"}, K: 3}, - {Query: "miss", Relevant: []string{"x"}, K: 3}, - {Query: "neg", Relevant: []string{}, K: 3}, - } - searchFn := func(query string, limit int) ([]string, error) { - switch query { - case "hit": - return []string{"a", "c", "b"}, nil - case "miss": - return []string{"y", "z"}, nil - case "neg": - return []string{"unexpected"}, nil - default: - return nil, nil - } - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if len(report.PerQuery) != 3 { - t.Fatalf("PerQuery len: got %d, want 3", len(report.PerQuery)) - } - if report.PerQuery[0].Query != "hit" || !approxEqual(report.PerQuery[0].PAt1, 1.0) || !approxEqual(report.PerQuery[0].MRR, 1.0) || report.PerQuery[0].ResultsReturned != 3 || report.PerQuery[0].Kind != "positive" { - t.Fatalf("hit diagnostic mismatch: %+v", report.PerQuery[0]) - } - if report.PerQuery[1].Query != "miss" || !approxEqual(report.PerQuery[1].PAt1, 0.0) || !approxEqual(report.PerQuery[1].MRR, 0.0) || report.PerQuery[1].Kind != "positive" { - t.Fatalf("miss diagnostic mismatch: %+v", report.PerQuery[1]) - } - if report.PerQuery[2].Query != "neg" || report.PerQuery[2].Kind != "negative" || !report.PerQuery[2].FalsePositive { - t.Fatalf("neg diagnostic mismatch: %+v", report.PerQuery[2]) - } -} - -func TestEvaluateQueries_AggregatesUnchangedAfterDiagnostics(t *testing.T) { - cases := []QueryCase{{Query: "Hello", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"a", "c", "b"}, nil } - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) { - t.Fatalf("aggregates regressed: %+v", report) - } -} - -func TestEvaluateQueries_NegativeDiagnosticIncludesReturnedKeys(t *testing.T) { - cases := []QueryCase{{Query: "neg", Relevant: []string{}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"unexpected1", "unexpected2"}, nil } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if len(report.PerQuery) != 1 { - t.Fatalf("PerQuery len: got %d, want 1", len(report.PerQuery)) - } - got := report.PerQuery[0].TopResults - want := []string{"unexpected1", "unexpected2"} - if !slices.Equal(got, want) { - t.Fatalf("TopResults: got %v, want %v", got, want) - } -} - -func TestEvaluateQueries_PositiveDiagnosticIncludesReturnedKeys(t *testing.T) { - cases := []QueryCase{{Query: "hit", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"a", "c", "b"}, nil } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if len(report.PerQuery) != 1 { - t.Fatalf("PerQuery len: got %d, want 1", len(report.PerQuery)) - } - got := report.PerQuery[0].TopResults - want := []string{"a", "c", "b"} - if !slices.Equal(got, want) { - t.Fatalf("TopResults: got %v, want %v", got, want) - } -} - -func TestEvaluateQueries_DiagnosticTopResultsCappedAtFive(t *testing.T) { - cases := []QueryCase{{Query: "hit", Relevant: []string{"a"}, K: 5}} - searchFn := func(string, int) ([]string, error) { - return []string{"a", "b", "c", "d", "e", "f", "g", "h"}, nil - } - - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - got := report.PerQuery[0].TopResults - want := []string{"a", "b", "c", "d", "e"} - if !slices.Equal(got, want) { - t.Fatalf("TopResults cap: got %v, want %v", got, want) - } -} - -func TestEvaluateQueries_AggregatesUnchangedAfterTopResults(t *testing.T) { - cases := []QueryCase{{Query: "Hello", Relevant: []string{"a", "b"}, K: 3}} - searchFn := func(string, int) ([]string, error) { return []string{"a", "c", "b"}, nil } - report, err := EvaluateQueries(cases, searchFn) - if err != nil { - t.Fatalf("EvaluateQueries: %v", err) - } - if !approxEqual(report.AvgPAt1, 1.0) || !approxEqual(report.AvgMRR, 1.0) || !approxEqual(report.AvgRecallAt5, 1.0) || !approxEqual(report.NegativePassRate, 0.0) { - t.Fatalf("aggregates regressed: %+v", report) - } -} diff --git a/internal/mcp/handler_analysis.go b/internal/mcp/handler_analysis.go index 787fadc..71cfed4 100644 --- a/internal/mcp/handler_analysis.go +++ b/internal/mcp/handler_analysis.go @@ -4,7 +4,6 @@ package mcp import ( "context" "fmt" - "path/filepath" "slices" "github.com/mark3labs/mcp-go/mcp" @@ -19,6 +18,7 @@ import ( "github.com/tae2089/code-context-graph/internal/model" "github.com/tae2089/code-context-graph/internal/obs" "github.com/tae2089/code-context-graph/internal/paging" + "github.com/tae2089/code-context-graph/internal/safepath" ) const ( @@ -614,7 +614,7 @@ func validateRepoRootWithin(repoRoot, configuredRepoRoot, namespaceRoot string) if len(allowedRoots) == 0 { return "", fmt.Errorf("analysis repo root is not configured") } - repo, err := canonicalPath(repoRoot) + repo, err := safepath.Canonical(repoRoot, false) if err != nil { return "", fmt.Errorf("invalid repo_root: %w", err) } @@ -653,11 +653,11 @@ func sliceContainsString(values []string, target string) bool { // @requires allowedRoots must be non-empty and each entry must be a valid filesystem path. func validatePathWithinAllowedRoots(target string, allowedRoots []string) (bool, error) { for _, root := range allowedRoots { - base, err := canonicalPath(root) + base, err := safepath.Canonical(root, false) if err != nil { return false, err } - within, err := isWithinRoot(base, target) + within, err := safepath.IsWithinRoot(base, target) if err != nil { return false, err } @@ -667,39 +667,3 @@ func validatePathWithinAllowedRoots(target string, allowedRoots []string) (bool, } return false, nil } - -// isWithinRoot reports whether target is the same as root or a descendant of it. -// @intent detect path traversal by checking the relative path does not escape upward. -func isWithinRoot(root, target string) (bool, error) { - rel, err := filepath.Rel(root, target) - if err != nil { - return false, err - } - if rel == "." { - return true, nil - } - if rel == ".." { - return false, nil - } - if len(rel) >= 3 && rel[:3] == ".."+string(filepath.Separator) { - return false, nil - } - if filepath.IsAbs(rel) { - return false, nil - } - return true, nil -} - -// canonicalPath resolves path to an absolute, symlink-free, cleaned filesystem path. -// @intent normalize user-supplied paths before comparison to prevent symlink-based boundary escapes. -func canonicalPath(path string) (string, error) { - abs, err := filepath.Abs(path) - if err != nil { - return "", err - } - real, err := filepath.EvalSymlinks(abs) - if err != nil { - return "", err - } - return filepath.Clean(real), nil -} diff --git a/internal/mcp/handler_docs.go b/internal/mcp/handler_docs.go index d9d1c3e..1dad43e 100644 --- a/internal/mcp/handler_docs.go +++ b/internal/mcp/handler_docs.go @@ -216,8 +216,10 @@ func (h *handlers) getDocContent(ctx context.Context, request mcp.CallToolReques return mcp.NewToolResultError("invalid file_path: path traversal not allowed"), nil } + // The default namespace maps to the shared docs root, mirroring resolvedRagIndexPath + // and retrieve_docs; only named namespaces resolve under namespaces//. var resolvedPath string - if namespace != "" { + if namespace != "" && ctxns.Normalize(namespace) != ctxns.DefaultNamespace { if err := validateNamespacePath(namespace, filePath); err != nil { return mcp.NewToolResultError(err.Error()), nil } @@ -326,8 +328,6 @@ type retrieveDocsResponse = retrieval.Response // @intent keep MCP retrieve_docs result decoding compatible while sharing the canonical retrieval DTO. type retrieveDocsResult = retrieval.Result -var _ = (*handlers).retrieveDocsFromDB - // @intent build a retrieve_docs response from persisted graph nodes and annotation tags. // @requires SearchBackend and DB must be configured, and ctx must carry the desired namespace. // @ensures returned results are grouped one-per-file in first-seen FTS order and keep retrieve_docs' stable JSON shape. diff --git a/internal/mcp/handler_docs_test.go b/internal/mcp/handler_docs_test.go index ab0f3a8..0e863c0 100644 --- a/internal/mcp/handler_docs_test.go +++ b/internal/mcp/handler_docs_test.go @@ -295,6 +295,33 @@ func TestGetDocContent_PathTraversal(t *testing.T) { } } +func TestGetDocContent_DefaultNamespaceReadsSharedDocs(t *testing.T) { + deps := setupTestDeps(t) + deps.RagIndexDir = t.TempDir() + + content := "# Shared Doc\nfrom shared docs root" + docPath := filepath.Join(deps.RagIndexDir, "docs", "shared-doc.md") + if err := os.MkdirAll(filepath.Dir(docPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(docPath, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + // "default" must resolve to the shared docs root, matching resolvedRagIndexPath + // and retrieve_docs (contentNamespace maps default to shared), not namespaces/default/. + result := callTool(t, deps, "get_doc_content", map[string]any{ + "namespace": "default", + "file_path": "docs/shared-doc.md", + }) + if result.IsError { + t.Fatalf("default namespace should read shared docs, got error: %v", getTextContent(result)) + } + if got := getTextContent(result); got != content { + t.Errorf("want %q, got %q", content, got) + } +} + func TestGetDocContent_NotFound(t *testing.T) { deps := setupTestDeps(t) result := callTool(t, deps, "get_doc_content", map[string]any{ diff --git a/internal/mcp/handler_parse.go b/internal/mcp/handler_parse.go index 76b07c4..7edc214 100644 --- a/internal/mcp/handler_parse.go +++ b/internal/mcp/handler_parse.go @@ -3,10 +3,7 @@ package mcp import ( "context" - "errors" "fmt" - "os" - "path/filepath" "slices" "time" @@ -17,6 +14,7 @@ import ( flowspkg "github.com/tae2089/code-context-graph/internal/analysis/flows" "github.com/tae2089/code-context-graph/internal/obs" postprocesspolicy "github.com/tae2089/code-context-graph/internal/postprocess/policy" + "github.com/tae2089/code-context-graph/internal/safepath" "github.com/tae2089/code-context-graph/internal/service" ) @@ -522,7 +520,7 @@ func (h *handlers) validateAnalysisPath(path string) (string, error) { if len(allowedRoots) == 0 { return "", fmt.Errorf("analysis root is not configured") } - target, err := canonicalExistingPath(path) + target, err := safepath.Canonical(path, true) if err != nil { return "", fmt.Errorf("invalid path: %w", err) } @@ -535,28 +533,3 @@ func (h *handlers) validateAnalysisPath(path string) (string, error) { } return target, nil } - -// @intent resolve a path to its absolute, symlink-evaluated form for analysis-root containment checks. -// @domainRule symlink evaluation must happen before containment checks against analysis roots. -// @ensures returned path is cleaned and absolute; existing paths are fully symlink-resolved. -func canonicalExistingPath(path string) (string, error) { - abs, err := filepath.Abs(path) - if err != nil { - return "", err - } - clean := filepath.Clean(abs) - real, err := filepath.EvalSymlinks(clean) - if err == nil { - return filepath.Clean(real), nil - } - if !errors.Is(err, os.ErrNotExist) { - return "", err - } - parent := filepath.Dir(clean) - base := filepath.Base(clean) - parentReal, parentErr := filepath.EvalSymlinks(parent) - if parentErr != nil { - return "", err - } - return filepath.Join(parentReal, base), nil -} diff --git a/internal/mcp/handlers_test.go b/internal/mcp/handlers_test.go index 6272e2c..423be8d 100644 --- a/internal/mcp/handlers_test.go +++ b/internal/mcp/handlers_test.go @@ -973,14 +973,12 @@ func TestBuildOrUpdateGraph_IncrementalIncludePaths_DefaultsToReplace(t *testing mockSync := &mockIncrementalSyncer{result: &incremental.SyncStats{}} deps.Incremental = mockSync + // Only handler.go is on disk; the out-of-scope other.go is missing. Under default replace + // semantics the delete pass must consider all namespace files, so other.go is deletable. dir := t.TempDir() os.MkdirAll(filepath.Join(dir, "src", "api"), 0o755) - os.MkdirAll(filepath.Join(dir, "src", "other"), 0o755) writeGoFile(t, filepath.Join(dir, "src", "api"), "handler.go", `package api func Handler() {} -`) - writeGoFile(t, filepath.Join(dir, "src", "other"), "other.go", `package other -func Other() {} `) callTool(t, deps, "build_or_update_graph", map[string]any{ @@ -994,16 +992,24 @@ func Other() {} if !mockSync.syncWithExisting { t.Fatal("expected Incremental.SyncWithExisting to be called") } - if len(mockSync.existingCalls) == 0 { - t.Fatal("expected at least one SyncWithExisting call") + deleteExisting := deletePassExistingFiles(mockSync) + if deleteExisting == nil { + t.Fatalf("expected a delete pass for the missing out-of-scope file, calls=%v", mockSync.existingCalls) } - firstExisting := mockSync.existingCalls[0] - if len(firstExisting) != 2 { - t.Fatalf("expected default replace semantics to pass all namespace files, got %v", firstExisting) + if !containsStringInSlice(deleteExisting, filepath.Join("src", "other", "other.go")) { + t.Fatalf("default replace must consider the out-of-scope file for deletion, got %v", deleteExisting) } - if !containsStringInSlice(firstExisting, filepath.Join("src", "other", "other.go")) { - t.Fatalf("expected existingFiles to include out-of-scope file under default replace semantics, got %v", firstExisting) +} + +// deletePassExistingFiles returns the existingFiles of the delete pass (the SyncWithExisting +// call that received no files), or nil when no delete pass ran. +func deletePassExistingFiles(m *mockIncrementalSyncer) []string { + for i := range m.filesCalls { + if len(m.filesCalls[i]) == 0 && len(m.existingCalls[i]) > 0 { + return m.existingCalls[i] + } } + return nil } func TestBuildOrUpdateGraph_IncrementalIncludePaths_ReplaceFalsePreservesOutOfScopeFiles(t *testing.T) { @@ -1020,14 +1026,13 @@ func TestBuildOrUpdateGraph_IncrementalIncludePaths_ReplaceFalsePreservesOutOfSc mockSync := &mockIncrementalSyncer{result: &incremental.SyncStats{}} deps.Incremental = mockSync + // The seeded handler.go and other.go are both missing from disk; a new in-scope file is + // added. With replace=false + include_paths, only the in-scope handler.go may be deleted, + // and the out-of-scope other.go must be scoped out of the existing-file set entirely. dir := t.TempDir() os.MkdirAll(filepath.Join(dir, "src", "api"), 0o755) - os.MkdirAll(filepath.Join(dir, "src", "other"), 0o755) - writeGoFile(t, filepath.Join(dir, "src", "api"), "handler.go", `package api -func Handler() {} -`) - writeGoFile(t, filepath.Join(dir, "src", "other"), "other.go", `package other -func Other() {} + writeGoFile(t, filepath.Join(dir, "src", "api"), "new.go", `package api +func New() {} `) callTool(t, deps, "build_or_update_graph", map[string]any{ @@ -1042,15 +1047,15 @@ func Other() {} if !mockSync.syncWithExisting { t.Fatal("expected Incremental.SyncWithExisting to be called") } - if len(mockSync.existingCalls) == 0 { - t.Fatal("expected at least one SyncWithExisting call") + deleteExisting := deletePassExistingFiles(mockSync) + if deleteExisting == nil { + t.Fatalf("expected a delete pass for the missing in-scope file, calls=%v", mockSync.existingCalls) } - firstExisting := mockSync.existingCalls[0] - if containsStringInSlice(firstExisting, filepath.Join("src", "other", "other.go")) { - t.Fatalf("expected replace=false to exclude out-of-scope file from existingFiles, got %v", firstExisting) + if containsStringInSlice(deleteExisting, filepath.Join("src", "other", "other.go")) { + t.Fatalf("replace=false must exclude the out-of-scope file from deletions, got %v", deleteExisting) } - if !containsStringInSlice(firstExisting, filepath.Join("src", "api", "handler.go")) { - t.Fatalf("expected replace=false to keep in-scope file, got %v", firstExisting) + if !containsStringInSlice(deleteExisting, filepath.Join("src", "api", "handler.go")) { + t.Fatalf("replace=false must still delete the missing in-scope file, got %v", deleteExisting) } } diff --git a/internal/namespacefs/workspace.go b/internal/namespacefs/workspace.go index fa5667e..cb91e29 100644 --- a/internal/namespacefs/workspace.go +++ b/internal/namespacefs/workspace.go @@ -11,6 +11,8 @@ import ( "path/filepath" "slices" "strings" + + "github.com/tae2089/code-context-graph/internal/safepath" ) // Default upload size limits applied when callers do not provide their own Limits. @@ -133,29 +135,7 @@ func ValidatePath(namespace, filePath string) error { // @intent prevent symlink traversal from escaping the namespace root before any filesystem mutation. // @param allowMissingLeaf when true, returns the joined path even when the leaf does not yet exist. func EnsureNoSymlinkInPath(root, relPath string, allowMissingLeaf bool) (string, error) { - cleanRel := filepath.Clean(relPath) - if cleanRel == "." { - return root, nil - } - current := root - segments := strings.Split(cleanRel, string(filepath.Separator)) - for i, segment := range segments { - current = filepath.Join(current, segment) - info, err := os.Lstat(current) - if err != nil { - if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) && i == len(segments)-1 { - return current, nil - } - if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) { - continue - } - return "", err - } - if info.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("symlink paths are not allowed") - } - } - return current, nil + return safepath.EnsureNoSymlinkInPath(root, relPath, allowMissingLeaf) } // SafeWrite atomically writes data to path using a temp file and rename, refusing to follow symlinks. diff --git a/internal/parse/binder_test.go b/internal/parse/binder_test.go index 9261d76..a0abbcf 100644 --- a/internal/parse/binder_test.go +++ b/internal/parse/binder_test.go @@ -111,10 +111,11 @@ func TestBinder_MultipleDeclarations(t *testing.T) { // 바인딩이 성공함을 검증합니다. // // fixture 기준: -// Line 1-4: docstring ("""...""") -// Line 5: @app.route('/api/user') -// Line 6: @login_required -// Line 7: def get_user(): ← tree-sitter StartLine 예상 +// +// Line 1-4: docstring ("""...""") +// Line 5: @app.route('/api/user') +// Line 6: @login_required +// Line 7: def get_user(): ← tree-sitter StartLine 예상 // // gap = 7 - 4 = 3 ≤ maxGap(3) → 바인딩 성공 func TestBinder_PythonDecoratorGap_BindsWithinMaxGap(t *testing.T) { @@ -188,11 +189,12 @@ func TestBinder_JavaAnnotationGap_PassthroughBinds(t *testing.T) { // 바인딩이 성공함을 검증합니다. // // fixture (attribute_gap.rs) 기준: -// Line 1: /// @intent 비동기 main 진입점 -// Line 2: /// @sideEffect 런타임 초기화 -// Line 3: #[tokio::main] -// Line 4: #[allow(dead_code)] -// Line 5: async fn main() { ← tree-sitter StartLine +// +// Line 1: /// @intent 비동기 main 진입점 +// Line 2: /// @sideEffect 런타임 초기화 +// Line 3: #[tokio::main] +// Line 4: #[allow(dead_code)] +// Line 5: async fn main() { ← tree-sitter StartLine // // gap = 5 - 2 = 3 ≤ maxGap(3) → 바인딩 성공 func TestBinder_RustAttributeGap_BindsWithinMaxGap(t *testing.T) { @@ -226,10 +228,11 @@ func TestBinder_RustAttributeGap_BindsWithinMaxGap(t *testing.T) { // 바인딩이 성공함을 검증합니다. // // fixture (attribute_gap.c) 기준: -// Line 1-3: /** ... */ Doxygen -// Line 4: __attribute__((always_inline)) -// Line 5: __attribute__((nonnull)) -// Line 6: static inline int add(int a, int b) { ← tree-sitter StartLine 예상 +// +// Line 1-3: /** ... */ Doxygen +// Line 4: __attribute__((always_inline)) +// Line 5: __attribute__((nonnull)) +// Line 6: static inline int add(int a, int b) { ← tree-sitter StartLine 예상 // // gap = 6 - 3 = 3 ≤ maxGap(3) → 바인딩 성공 func TestBinder_CAttributeGap_BindsWithinMaxGap(t *testing.T) { @@ -292,7 +295,6 @@ func TestBinder_GapWithCodeBetween_NoBinding(t *testing.T) { } } - // ============================================================================= // Look-Between 동적 바인딩 테스트 // ============================================================================= @@ -320,10 +322,10 @@ func TestBinder_LookBetween_BlankLinesOnly_Binds(t *testing.T) { } sourceLines := []string{ "// @intent 빈 줄 사이 바인딩", // line 1 - "", // line 2 - "", // line 3 - "", // line 4 - "func MyFunc() {", // line 5 + "", // line 2 + "", // line 3 + "", // line 4 + "func MyFunc() {", // line 5 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -357,8 +359,8 @@ func TestBinder_LookBetween_CodeBetween_NoBinding(t *testing.T) { } sourceLines := []string{ "// @intent 이 주석은 바인딩 안 됨", // line 1 - "var x = 42", // line 2 - 코드! - "func MyFunc() {", // line 3 + "var x = 42", // line 2 - 코드! + "func MyFunc() {", // line 3 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -395,8 +397,8 @@ func TestBinder_LookBetween_Adjacent_Gap1_Binds(t *testing.T) { } sourceLines := []string{ "// @intent 인접 바인딩", // line 1 - "// (continued)", // line 2 - "func MyFunc() {", // line 3 + "// (continued)", // line 2 + "func MyFunc() {", // line 3 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -439,9 +441,9 @@ func TestBinder_LookBetween_WhitespaceOnlyLines_Binds(t *testing.T) { } sourceLines := []string{ "// @intent 공백문자만 있는 줄", // line 1 - " ", // line 2 - spaces only - "\t\t", // line 3 - tabs only - "func MyFunc() {", // line 4 + " ", // line 2 - spaces only + "\t\t", // line 3 - tabs only + "func MyFunc() {", // line 4 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -477,9 +479,9 @@ func TestBinder_Passthrough_PythonDecorators_Binds(t *testing.T) { } sourceLines := []string{ "# @intent 사용자 조회 엔드포인트", // line 1 - "@app.route('/api/user')", // line 2 - decorator - "@login_required", // line 3 - decorator - "def get_user():", // line 4 + "@app.route('/api/user')", // line 2 - decorator + "@login_required", // line 3 - decorator + "def get_user():", // line 4 } bindings := b.Bind(comments, nodes, "python", sourceLines) @@ -511,13 +513,13 @@ func TestBinder_Passthrough_JavaAnnotations_Binds(t *testing.T) { {Name: "UserService", Kind: model.NodeKindClass, StartLine: 7, EndLine: 20}, } sourceLines := []string{ - "/**", // line 1 - " * @intent 사용자 서비스", // line 2 - " */", // line 3 - "@Service", // line 4 - annotation - "@Transactional", // line 5 - annotation - "@RequiredArgsConstructor", // line 6 - annotation - "public class UserService {", // line 7 + "/**", // line 1 + " * @intent 사용자 서비스", // line 2 + " */", // line 3 + "@Service", // line 4 - annotation + "@Transactional", // line 5 - annotation + "@RequiredArgsConstructor", // line 6 - annotation + "public class UserService {", // line 7 } bindings := b.Bind(comments, nodes, "java", sourceLines) @@ -549,11 +551,11 @@ func TestBinder_Passthrough_RustAttributes_Binds(t *testing.T) { {Name: "main", Kind: model.NodeKindFunction, StartLine: 5, EndLine: 10}, } sourceLines := []string{ - "/// @intent 비동기 main 진입점", // line 1 - "/// @sideEffect 런타임 초기화", // line 2 - "#[tokio::main]", // line 3 - Rust attribute - "#[allow(dead_code)]", // line 4 - Rust attribute - "async fn main() {", // line 5 + "/// @intent 비동기 main 진입점", // line 1 + "/// @sideEffect 런타임 초기화", // line 2 + "#[tokio::main]", // line 3 - Rust attribute + "#[allow(dead_code)]", // line 4 - Rust attribute + "async fn main() {", // line 5 } bindings := b.Bind(comments, nodes, "rust", sourceLines) @@ -584,12 +586,12 @@ func TestBinder_Passthrough_CAttributes_Binds(t *testing.T) { {Name: "add", Kind: model.NodeKindFunction, StartLine: 6, EndLine: 10}, } sourceLines := []string{ - "/**", // line 1 - " * @intent 항상 인라인되는 덧셈", // line 2 - " */", // line 3 - "__attribute__((always_inline))", // line 4 - C attribute - "[[nodiscard]]", // line 5 - C++17 attribute - "static inline int add(int a, int b) {", // line 6 + "/**", // line 1 + " * @intent 항상 인라인되는 덧셈", // line 2 + " */", // line 3 + "__attribute__((always_inline))", // line 4 - C attribute + "[[nodiscard]]", // line 5 - C++17 attribute + "static inline int add(int a, int b) {", // line 6 } bindings := b.Bind(comments, nodes, "c", sourceLines) @@ -620,10 +622,10 @@ func TestBinder_Passthrough_OtherComments_Binds(t *testing.T) { {Name: "MyFunc", Kind: model.NodeKindFunction, StartLine: 4, EndLine: 10}, } sourceLines := []string{ - "// @intent 주석 사이 바인딩", // line 1 - "// TODO: 나중에 리팩토링", // line 2 - comment + "// @intent 주석 사이 바인딩", // line 1 + "// TODO: 나중에 리팩토링", // line 2 - comment "// NOTE: 이 함수는 deprecated 예정", // line 3 - comment - "func MyFunc() {", // line 4 + "func MyFunc() {", // line 4 } bindings := b.Bind(comments, nodes, "go", sourceLines) @@ -657,11 +659,11 @@ func TestBinder_Passthrough_Mixed_Binds(t *testing.T) { } sourceLines := []string{ "# @intent 혼합 passthrough 테스트", // line 1 - "# type: ignore", // line 2 - comment - "", // line 3 - blank - "@app.route('/test')", // line 4 - decorator - "@requires_auth", // line 5 - decorator - "def handler():", // line 6 + "# type: ignore", // line 2 - comment + "", // line 3 - blank + "@app.route('/test')", // line 4 - decorator + "@requires_auth", // line 5 - decorator + "def handler():", // line 6 } bindings := b.Bind(comments, nodes, "python", sourceLines) @@ -693,9 +695,9 @@ func TestBinder_Passthrough_DecoratorPlusCode_NoBinding(t *testing.T) { } sourceLines := []string{ "# @intent 이건 바인딩 안 됨", // line 1 - "@decorator", // line 2 - decorator (passthrough) - "x = 42", // line 3 - actual code! - "def handler():", // line 4 + "@decorator", // line 2 - decorator (passthrough) + "x = 42", // line 3 - actual code! + "def handler():", // line 4 } bindings := b.Bind(comments, nodes, "python", sourceLines) diff --git a/internal/parse/treesitter/binding_gap_cross_language_test.go b/internal/parse/treesitter/binding_gap_cross_language_test.go index 878fa87..69dfd46 100644 --- a/internal/parse/treesitter/binding_gap_cross_language_test.go +++ b/internal/parse/treesitter/binding_gap_cross_language_test.go @@ -9,8 +9,8 @@ // 테스트 정책: // - expectBound=true → Green 계약. 정상 바인딩되어야 하며, 실패 시 회귀(normalizer/walker/binder). // - expectBound=false → Red 계약. 현재 tree-sitter grammar 특성으로 바인딩이 누락됨을 -// "명시적으로" 고정. 구현이 개선되어 우연히 바인딩되면 테스트가 실패하므로 -// 그때 Green으로 승격하고 skipReason과 walker 보정을 함께 정리한다. +// "명시적으로" 고정. 구현이 개선되어 우연히 바인딩되면 테스트가 실패하므로 +// 그때 Green으로 승격하고 skipReason과 walker 보정을 함께 정리한다. // // Helper 함수: // - binderFromWalkerComments, logNodeInfo, logCommentInfo 는 @@ -338,10 +338,10 @@ data class User(val name: String) } // Phase 2: 바인딩 시도. - binder := parse.NewBinder() - bindings := binder.Bind( - binderFromWalkerComments(walkerComments), nodes, tc.spec.Name, strings.Split(tc.source, "\n"), - ) + binder := parse.NewBinder() + bindings := binder.Bind( + binderFromWalkerComments(walkerComments), nodes, tc.spec.Name, strings.Split(tc.source, "\n"), + ) var target *parse.Binding for i := range bindings { diff --git a/internal/parse/treesitter/binding_gap_go_directives_test.go b/internal/parse/treesitter/binding_gap_go_directives_test.go index 42da704..e39d1b9 100644 --- a/internal/parse/treesitter/binding_gap_go_directives_test.go +++ b/internal/parse/treesitter/binding_gap_go_directives_test.go @@ -68,8 +68,8 @@ func TestWalkerBinder_Go_DirectivePollution(t *testing.T) { t.Log("--- 파싱된 CommentBlock ---") logCommentInfo(t, walkerComments) - b := parse.NewBinder() - bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, "go", strings.Split(string(content), "\n")) + b := parse.NewBinder() + bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, "go", strings.Split(string(content), "\n")) var intentValue string var bound bool diff --git a/internal/parse/treesitter/binding_gap_integration_test.go b/internal/parse/treesitter/binding_gap_integration_test.go index f693c3c..c14cf38 100644 --- a/internal/parse/treesitter/binding_gap_integration_test.go +++ b/internal/parse/treesitter/binding_gap_integration_test.go @@ -4,9 +4,9 @@ // 여러 줄 붙어 있으면 @intent docstring/주석이 심볼 노드에 바인딩되지 않는다. // // 이 파일의 테스트는 각 언어별로: -// 1. 실제 tree-sitter로 fixture를 파싱 -// 2. 심볼 노드의 실제 StartLine을 t.Logf로 출력 -// 3. Binder.Bind 결과를 검증하여 현재 실패 동작을 명시적으로 기록 +// 1. 실제 tree-sitter로 fixture를 파싱 +// 2. 심볼 노드의 실제 StartLine을 t.Logf로 출력 +// 3. Binder.Bind 결과를 검증하여 현재 실패 동작을 명시적으로 기록 // // 테스트 이름에 _CurrentlyFailsBinding 접미사를 붙여 현재 상태를 명시합니다. package treesitter @@ -421,15 +421,17 @@ func TestWalkerBinder_CAttribute_CurrentlyFailsBinding(t *testing.T) { // # 라인 주석 + 데코레이터 2개 시나리오로 바인딩 실패를 검증합니다. // // fixture (decorator_gap_comment.py) 기준: -// Line 1: # @intent 사용자 조회 엔드포인트 -// Line 2: # @domainRule 인증 필요 -// Line 3: @app.route('/api/user') -// Line 4: @login_required -// Line 5: def get_user(): ← tree-sitter StartLine +// +// Line 1: # @intent 사용자 조회 엔드포인트 +// Line 2: # @domainRule 인증 필요 +// Line 3: @app.route('/api/user') +// Line 4: @login_required +// Line 5: def get_user(): ← tree-sitter StartLine // // 주석EndLine=2, get_user StartLine=5, gap=3 > maxGap(2) → 바인딩 실패 예상 // (단, Python이 decorated_definition으로 StartLine을 데코레이터 첫 줄=3으로 잡으면 -// gap=3-2=1이 되어 바인딩 성공 가능성도 있음) +// +// gap=3-2=1이 되어 바인딩 성공 가능성도 있음) func TestWalkerBinder_PythonDecoratorHashComment_CurrentlyFailsBinding(t *testing.T) { content := readFixture(t, "python", "decorator_gap_comment.py") t.Logf("fixture 내용:\n%s", string(content)) @@ -517,11 +519,11 @@ func TestWalkerBinder_PythonDecoratorHashComment_CurrentlyFailsBinding(t *testin // 이 테스트는 항상 통과하며, 진단 데이터만 수집합니다. func TestWalkerBinder_GapDiagnosis_AllLanguages(t *testing.T) { type langCase struct { - spec *LangSpec - lang string - filename string - symName string - symKind model.NodeKind + spec *LangSpec + lang string + filename string + symName string + symKind model.NodeKind fixtureExt string } diff --git a/internal/parse/treesitter/binding_gap_p1_test.go b/internal/parse/treesitter/binding_gap_p1_test.go index ae3feb9..4d5f196 100644 --- a/internal/parse/treesitter/binding_gap_p1_test.go +++ b/internal/parse/treesitter/binding_gap_p1_test.go @@ -9,10 +9,10 @@ // - 언어별 tree-sitter 문법이 메타 표식을 심볼 노드에 포함하는지 확인 // // 검증 대상 4개 분류: -// 1. 심볼 StartLine이 메타 표식 줄을 포함하는가 (Java/C처럼 wrapper가 흡수) -// 2. @intent 태그가 정상 파싱되는가 (normalizer 이슈 없는가) -// 3. gap이 maxGap(=2) 이내인가 -// 4. 최종적으로 Binder가 @intent를 바인딩하는가 +// 1. 심볼 StartLine이 메타 표식 줄을 포함하는가 (Java/C처럼 wrapper가 흡수) +// 2. @intent 태그가 정상 파싱되는가 (normalizer 이슈 없는가) +// 3. gap이 maxGap(=2) 이내인가 +// 4. 최종적으로 Binder가 @intent를 바인딩하는가 package treesitter import ( @@ -215,11 +215,11 @@ func TestWalkerBinder_P1_AllLanguages_Summary(t *testing.T) { } type row struct { - label string - startLine int - intentBound bool - minGap int - found bool + label string + startLine int + intentBound bool + minGap int + found bool } var rows []row @@ -252,9 +252,9 @@ func TestWalkerBinder_P1_AllLanguages_Summary(t *testing.T) { } } - // 바인딩 체크 - b := parse.NewBinder() - bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, tc.lang, strings.Split(string(content), "\n")) + // 바인딩 체크 + b := parse.NewBinder() + bindings := b.Bind(binderFromWalkerComments(walkerComments), nodes, tc.lang, strings.Split(string(content), "\n")) for _, binding := range bindings { if binding.Node.Name != tc.symName || binding.Node.Kind != tc.symKind { continue diff --git a/internal/parse/treesitter/p1_ast_probe_test.go b/internal/parse/treesitter/p1_ast_probe_test.go index 4ba4e87..a95cc6e 100644 --- a/internal/parse/treesitter/p1_ast_probe_test.go +++ b/internal/parse/treesitter/p1_ast_probe_test.go @@ -69,15 +69,15 @@ func TestP1_AST_NodeTypes(t *testing.T) { } for _, tc := range cases { - t.Run(tc.label, func(t *testing.T) { - content := readFixture(t, tc.lang, tc.filename) - w := NewWalker(tc.spec) - parser := w.acquireParser() - defer w.releaseParser(parser) - tree, err := parser.ParseCtx(context.Background(), nil, content) - if err != nil { - t.Fatalf("[%s] 파싱 실패: %v", tc.label, err) - } + t.Run(tc.label, func(t *testing.T) { + content := readFixture(t, tc.lang, tc.filename) + w := NewWalker(tc.spec) + parser := w.acquireParser() + defer w.releaseParser(parser) + tree, err := parser.ParseCtx(context.Background(), nil, content) + if err != nil { + t.Fatalf("[%s] 파싱 실패: %v", tc.label, err) + } defer tree.Close() root := tree.RootNode() diff --git a/internal/parse/treesitter/package_discovery.go b/internal/parse/treesitter/package_discovery.go index 8ab8f94..7120e69 100644 --- a/internal/parse/treesitter/package_discovery.go +++ b/internal/parse/treesitter/package_discovery.go @@ -81,7 +81,7 @@ func (PythonPackageDiscovery) DiscoverPackages(ctx context.Context, opts Package // @intent create package nodes for package.json paths and tsconfig alias paths that imports can target. func (TypeScriptPackageDiscovery) DiscoverPackages(ctx context.Context, opts PackageDiscoveryOptions) (map[string]PackageInfo, error) { return discoverNodePackages(ctx, opts, nodePackageDiscoveryConfig{ - language: "typescript", + language: "typescript", extensions: []string{".ts", ".tsx"}, packageJSON: true, tsconfigAlias: true, @@ -92,7 +92,7 @@ func (TypeScriptPackageDiscovery) DiscoverPackages(ctx context.Context, opts Pac // @intent create package nodes for JavaScript directories using package.json-derived import paths. func (JavaScriptPackageDiscovery) DiscoverPackages(ctx context.Context, opts PackageDiscoveryOptions) (map[string]PackageInfo, error) { return discoverNodePackages(ctx, opts, nodePackageDiscoveryConfig{ - language: "javascript", + language: "javascript", extensions: []string{".js", ".jsx", ".mjs", ".cjs"}, packageJSON: true, }) @@ -349,7 +349,7 @@ func pathBaseName(value, sep string) string { // nodePackageDiscoveryConfig describes one Node.js-family package discovery mode. // @intent share package.json and tsconfig-based directory discovery across TypeScript and JavaScript. type nodePackageDiscoveryConfig struct { - language string + language string extensions []string packageJSON bool tsconfigAlias bool @@ -358,7 +358,7 @@ type nodePackageDiscoveryConfig struct { // nodePackageJSON is the subset of package.json used for import-path discovery. // @intent keep package metadata parsing minimal while deriving package-node qualified names. type nodePackageJSON struct { - Name string `json:"name"` + Name string `json:"name"` Workspaces json.RawMessage `json:"workspaces"` } diff --git a/internal/parse/treesitter/package_discovery_test.go b/internal/parse/treesitter/package_discovery_test.go index 208686a..63d529c 100644 --- a/internal/parse/treesitter/package_discovery_test.go +++ b/internal/parse/treesitter/package_discovery_test.go @@ -756,8 +756,7 @@ func TestPackageDiscoveryOrDefault_DefaultsForUnimplementedLanguages(t *testing. tests := []struct { name string spec *LangSpec - }{ - } + }{} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/parse/treesitter/python_docstring_variants_test.go b/internal/parse/treesitter/python_docstring_variants_test.go index 1b0e656..809bbb5 100644 --- a/internal/parse/treesitter/python_docstring_variants_test.go +++ b/internal/parse/treesitter/python_docstring_variants_test.go @@ -1,6 +1,6 @@ // Python docstring 2차 실측 테스트. // -// 목적: tree-sitter-python이 다양한 docstring 변형(함수/클래스/모듈, """/''', 한 줄, prefix)을 +// 목적: tree-sitter-python이 다양한 docstring 변형(함수/클래스/모듈, """/”', 한 줄, prefix)을 // 어떤 AST 노드 구조로 파싱하는지 정확히 측정하고, // 현재 collectComments 로직이 이들을 CommentBlock으로 수집하는지 검증한다. // @@ -145,13 +145,13 @@ func parseRawTree(t *testing.T, content []byte) *sitter.Node { // stringNodeDetail은 string 노드의 상세 정보를 분석한다. type stringNodeDetail struct { - nodeType string // "string", "string_content", 기타 - startLine int - endLine int - isOneLiner bool - prefixToken string // "r", "f", "b", "rb" 등 (없으면 "") - quoteStyle string // `"""` or `'''` or `"` or `'` - rawContent string // 노드 Content의 첫 80자 + nodeType string // "string", "string_content", 기타 + startLine int + endLine int + isOneLiner bool + prefixToken string // "r", "f", "b", "rb" 등 (없으면 "") + quoteStyle string // `"""` or `'''` or `"` or `'` + rawContent string // 노드 Content의 첫 80자 } func analyzeStringNode(node *sitter.Node, content []byte) stringNodeDetail { @@ -224,17 +224,17 @@ func analyzeStringNode(node *sitter.Node, content []byte) stringNodeDetail { // ───────────────────────────────────────────────────────────── type docstringMeasurement struct { - fixture string - nodeType string // string 노드 타입 - parentChain []string // 부모 체인 (루트 방향) - startLine int - endLine int - prefixToken string - quoteStyle string - isOneLiner bool - inCommentBlock bool // 현재 CommentBlock에 포함되는가 - intentBound bool // @intent 바인딩 성공 여부 - bindingNote string // 추가 메모 + fixture string + nodeType string // string 노드 타입 + parentChain []string // 부모 체인 (루트 방향) + startLine int + endLine int + prefixToken string + quoteStyle string + isOneLiner bool + inCommentBlock bool // 현재 CommentBlock에 포함되는가 + intentBound bool // @intent 바인딩 성공 여부 + bindingNote string // 추가 메모 } func (m docstringMeasurement) parentChainStr() string { @@ -253,7 +253,7 @@ func (m docstringMeasurement) parentChainStr() string { func measureDocstringFixture( t *testing.T, filename string, - targetSymName string, // 바인딩 대상 심볼 이름 ("" 이면 file 노드) + targetSymName string, // 바인딩 대상 심볼 이름 ("" 이면 file 노드) targetSymKind model.NodeKind, ) docstringMeasurement { t.Helper() @@ -408,10 +408,10 @@ func TestPythonDocstring_FuncDouble(t *testing.T) { } } -// TestPythonDocstring_FuncSingle 은 함수 본문 내 '''...''' docstring을 실측한다. +// TestPythonDocstring_FuncSingle 은 함수 본문 내 ”'...”' docstring을 실측한다. // // 검증 포인트: -// - """와 '''가 동일한 tree-sitter 노드 타입("string")으로 처리되는가 +// - """와 ”'가 동일한 tree-sitter 노드 타입("string")으로 처리되는가 func TestPythonDocstring_FuncSingle(t *testing.T) { m := measureDocstringFixture(t, "docstring_func_single.py", "get_user", model.NodeKindFunction) @@ -596,10 +596,10 @@ func TestPythonDocstring_Prefix(t *testing.T) { // 이 테스트 자체는 항상 Pass (진단 전용). func TestPythonDocstring_AllVariants_Summary(t *testing.T) { type fixtureCase struct { - filename string - symName string - symKind model.NodeKind - label string + filename string + symName string + symKind model.NodeKind + label string } cases := []fixtureCase{ @@ -611,14 +611,14 @@ func TestPythonDocstring_AllVariants_Summary(t *testing.T) { } type row struct { - label string - nodeType string - parentChain string - lines string - prefix string - quoteStyle string - inComment bool - intentBound bool + label string + nodeType string + parentChain string + lines string + prefix string + quoteStyle string + inComment bool + intentBound bool } var rows []row @@ -658,10 +658,10 @@ func TestPythonDocstring_AllVariants_Summary(t *testing.T) { r.lines = "-" } - // 바인딩 확인 - b := parse.NewBinder() - binderComments := binderFromWalkerComments(walkerComments) - bindings := b.Bind(binderComments, nodes, "python", strings.Split(string(content), "\n")) + // 바인딩 확인 + b := parse.NewBinder() + binderComments := binderFromWalkerComments(walkerComments) + bindings := b.Bind(binderComments, nodes, "python", strings.Split(string(content), "\n")) for _, binding := range bindings { var match bool if tc.symName == "" { @@ -732,10 +732,10 @@ func TestPythonDocstring_AllVariants_Summary(t *testing.T) { // (구현 수정 없이, 단순히 gap 계산 로직을 이해하기 위한 측정) func TestPythonDocstring_GapAnalysis(t *testing.T) { type gapCase struct { - name string - filename string - symName string - symKind model.NodeKind + name string + filename string + symName string + symKind model.NodeKind // docstring이 CommentBlock으로 주입될 때 예상 StartLine, EndLine fakeCommentStart int fakeCommentEnd int diff --git a/internal/parse/treesitter/walker.go b/internal/parse/treesitter/walker.go index ed7dd52..97cb165 100644 --- a/internal/parse/treesitter/walker.go +++ b/internal/parse/treesitter/walker.go @@ -656,24 +656,48 @@ func (w *Walker) inferEnclosingReceiver(defNode *sitter.Node, content []byte) st // mapDefTypeToNodeKind maps query definition labels to internal node kinds. // @intent keep language query captures aligned with graph node categorization -// @domainRule declarations matching the configured test prefix become test nodes +// @domainRule only function/method declarations whose name is a test name become test nodes; +// types, classes, and interfaces are never tests even when their name starts with the prefix func (w *Walker) mapDefTypeToNodeKind(defType string, name string) model.NodeKind { - if w.spec.TestPrefix != "" && strings.HasPrefix(name, w.spec.TestPrefix) { - return model.NodeKindTest - } - switch defType { case "class": return model.NodeKindClass case "interface", "type": return model.NodeKindType - case "function", "method": - return model.NodeKindFunction default: + if isTestName(name, w.spec.TestPrefix) { + return model.NodeKindTest + } return model.NodeKindFunction } } +// isTestName reports whether name denotes a test given the language's test prefix. +// @intent avoid misclassifying production symbols like testimonialCard or TestConfig as tests. +// @domainRule a separator-terminated prefix (e.g. "test_") is a boundary on its own; a bare-word +// prefix (e.g. "Test", "test", "TEST") must be followed by end-of-name or a non-lowercase character +// so it does not swallow a longer lowercase word. +func isTestName(name, prefix string) bool { + if prefix == "" || !strings.HasPrefix(name, prefix) { + return false + } + last := prefix[len(prefix)-1] + if !isASCIILetterOrDigit(last) { + return true + } + rest := name[len(prefix):] + if rest == "" { + return true + } + c := rest[0] + return c < 'a' || c > 'z' +} + +// @intent classify ASCII identifier characters for test-prefix boundary detection. +func isASCIILetterOrDigit(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + // buildQualifiedName joins package, receiver, and declaration name into a stable identifier. // @intent generate graph keys that distinguish methods from package-level declarations func (w *Walker) buildQualifiedName(pkg, receiver, name string) string { diff --git a/internal/parse/treesitter/walker_test.go b/internal/parse/treesitter/walker_test.go index b5e229c..1dc7980 100644 --- a/internal/parse/treesitter/walker_test.go +++ b/internal/parse/treesitter/walker_test.go @@ -1681,6 +1681,58 @@ func TestAdd(t *testing.T) {} } } +func TestParseGo_TestPrefixDoesNotMisclassifyProductionDecls(t *testing.T) { + src := `package main + +type TestConfig struct{} + +func Testimony() {} + +func TestAdd(t *testing.T) {} +` + w := NewWalker(GoSpec) + nodes, _, err := w.Parse("main_test.go", []byte(src)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + testNodes := filterByKind(nodes, model.NodeKindTest) + if len(testNodes) != 1 || testNodes[0].Name != "TestAdd" { + t.Fatalf("expected only TestAdd as a test node, got %+v", testNodes) + } + // TestConfig (a type) and Testimony (Test + lowercase 'i') must not be tests. + byName := map[string]model.NodeKind{} + for _, n := range nodes { + byName[n.Name] = n.Kind + } + if byName["TestConfig"] == model.NodeKindTest { + t.Errorf("TestConfig type must not be a test node") + } + if byName["Testimony"] == model.NodeKindTest { + t.Errorf("Testimony (lowercase continuation) must not be a test node") + } +} + +func TestParseTypeScript_TestPrefixRequiresWordBoundary(t *testing.T) { + src := `function testimonialCard() {} +function testLogin() {} +` + w := NewWalker(TypeScriptSpec) + nodes, _, err := w.Parse("app.ts", []byte(src)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + kinds := map[string]model.NodeKind{} + for _, n := range nodes { + kinds[n.Name] = n.Kind + } + if kinds["testimonialCard"] == model.NodeKindTest { + t.Errorf("testimonialCard must not be classified as a test") + } + if kinds["testLogin"] != model.NodeKindTest { + t.Errorf("testLogin should be classified as a test, got %v", kinds["testLogin"]) + } +} + func TestParsePython_TestFunction(t *testing.T) { src := `def test_add(): assert 1 + 1 == 2 diff --git a/internal/pathutil/exclude_test.go b/internal/pathutil/exclude_test.go index 93f5333..329dc1f 100644 --- a/internal/pathutil/exclude_test.go +++ b/internal/pathutil/exclude_test.go @@ -73,7 +73,7 @@ func TestLoadIncludePathsFromConfig_InvalidYAMLReturnsError(t *testing.T) { func TestMatchIncludePaths(t *testing.T) { tests := []struct { name string - relPath string + relPath string includePaths []string want bool }{ diff --git a/internal/retrieval/retrieval.go b/internal/retrieval/retrieval.go index 42ff326..50d7405 100644 --- a/internal/retrieval/retrieval.go +++ b/internal/retrieval/retrieval.go @@ -36,9 +36,8 @@ type Result struct { const ( dbCandidateFloor = 50 dbCandidateCap = 500 -) - -var ( - _ = dbCandidateFloor - _ = dbCandidateCap + // scanRowCap bounds the fallback namespace scan so a sparse-FTS query cannot load an + // unbounded number of nodes+annotations. Namespaces with fewer retrievable nodes than + // this behave identically to an uncapped scan; larger ones are truncated in stable order. + scanRowCap = 5000 ) diff --git a/internal/retrieval/service.go b/internal/retrieval/service.go index 853874d..eea7010 100644 --- a/internal/retrieval/service.go +++ b/internal/retrieval/service.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "sort" "strings" @@ -46,8 +47,15 @@ func (s *Service) FromDBWithOptions(ctx context.Context, namespace, query string candidateGroupLimit := DBCandidateLimit(limit) candidates := s.searchCandidates(ctx, query, limit) + // Capture the search engine's per-file rank order before the scan supplement is merged in, + // so engine hits keep their relevance ordering and outrank scan-only supplements. + ftsRanks := ftsFileRanks(candidates) groups, nodeIDs := GroupCandidatesByFile(candidates, candidateGroupLimit) - if len(groups) < candidateGroupLimit { + // @intent supplement with a DB scan only when FTS underfills the caller's requested + // result count, not whenever it returns fewer than the wide candidate ceiling. + // @domainRule scanning the whole namespace on every query makes retrieval O(namespace); + // gating on limit keeps the scan a genuine fallback for sparse-FTS queries. + if len(groups) < limit { scanned, err := s.scanDBCandidates(ctx, effectiveNamespace, query) if err != nil { return response, err @@ -70,7 +78,7 @@ func (s *Service) FromDBWithOptions(ctx context.Context, namespace, query string result.Matches = DBMatches(group.Nodes, annotations) response.Results = append(response.Results, result) } - sortRetrieveResults(response.Results) + sortRetrieveResults(response.Results, ftsRanks) if len(response.Results) > limit { response.Results = response.Results[:limit] } @@ -115,9 +123,15 @@ func (s *Service) scanDBCandidates(ctx context.Context, namespace, query string) Where("kind IN ?", retrievableNodeKinds). Preload("Annotation.Tags"). Order("file_path ASC, qualified_name ASC, id ASC"). + Limit(scanRowCap). Find(&nodes).Error; err != nil { return nil, fmt.Errorf("retrieve docs DB candidates: %w", err) } + if len(nodes) == scanRowCap { + // The scan hit its ceiling: matching nodes sorting after the cap are not considered. + slog.WarnContext(ctx, "retrieve_docs fallback scan truncated at cap; some matches may be omitted", + "namespace", namespace, "cap", scanRowCap) + } if len(nodes) == 0 { return nil, nil } @@ -159,9 +173,42 @@ func mergeCandidates(primary, supplemental []model.Node) []model.Node { return merged } -// @intent keep DB retrieve ordering score-first after all candidates have been structurally scored. -func sortRetrieveResults(results []Result) { +// ftsFileRanks records each file's best (earliest) position in the search engine's +// rank-ordered candidate list, keyed by file path. +// @intent preserve the search backend's relevance ordering as an authoritative ranking signal. +func ftsFileRanks(candidates []model.Node) map[string]int { + ranks := make(map[string]int, len(candidates)) + for pos, node := range candidates { + if !IsRetrievableNodeKind(node.Kind) { + continue + } + filePath := strings.TrimSpace(node.FilePath) + if filePath == "" { + continue + } + if _, seen := ranks[filePath]; !seen { + ranks[filePath] = pos + } + } + return ranks +} + +// @intent order DB retrieve results engine-first: search-backend hits keep their relevance rank +// and outrank scan-only supplements, with the structured annotation score as the refining signal. +func sortRetrieveResults(results []Result, ftsRanks map[string]int) { + rankOf := func(r Result) (int, bool) { + pos, ok := ftsRanks[strings.TrimPrefix(r.ID, "file:")] + return pos, ok + } sort.SliceStable(results, func(i, j int) bool { + ri, iHit := rankOf(results[i]) + rj, jHit := rankOf(results[j]) + if iHit != jHit { + return iHit // engine hits before scan-only supplements + } + if iHit && ri != rj { + return ri < rj // earlier engine rank wins + } if results[i].Score != results[j].Score { return results[i].Score > results[j].Score } diff --git a/internal/retrieval/service_test.go b/internal/retrieval/service_test.go index 327d711..2496df8 100644 --- a/internal/retrieval/service_test.go +++ b/internal/retrieval/service_test.go @@ -199,6 +199,59 @@ func TestServiceFromDB_supplementsPartialSearchResultsWithDBScan(t *testing.T) { } } +func TestServiceFromDB_skipsScanWhenSearchFillsLimit(t *testing.T) { + db := newRetrievalDB(t) + // FTS returns only a weak annotation-text match for a.go. + ftsNode := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Handler", Kind: model.NodeKindFunction, Name: "Handler", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, ftsNode.ID, "auth backend") + // scanOnly would score HIGHER (exact label match) but is only reachable via a DB scan; + // if the scan wrongly fires when FTS already fills the limit, this node would win and surface. + scanOnly := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Auth", Kind: model.NodeKindFunction, Name: "auth", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, scanOnly.ID, "auth service") + service := retrieval.Service{DB: db, SearchBackend: &stubSearchBackend{nodes: []model.Node{ftsNode}}} + + response, err := service.FromDB(context.Background(), "default", "auth", 1, 0, nil) + if err != nil { + t.Fatalf("FromDB returned error: %v", err) + } + if len(response.Results) != 1 { + t.Fatalf("expected one result, got %d: %+v", len(response.Results), response.Results) + } + if response.Results[0].ID != "file:a.go" { + t.Fatalf("expected FTS result only, got %q", response.Results[0].ID) + } + for _, r := range response.Results { + if r.ID == "file:b.go" { + t.Fatalf("scan-only node must not appear when FTS already fills the limit") + } + } +} + +func TestServiceFromDB_ftsHitOutranksHigherScoredScanSupplement(t *testing.T) { + db := newRetrievalDB(t) + // FTS returns a.go with only a weak annotation-text match. + weakFTS := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Handler", Kind: model.NodeKindFunction, Name: "Handler", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, weakFTS.ID, "auth request") + // b.go is reachable only via the scan supplement and scores much higher (exact label + high-weight tags). + strongScan := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.Auth", Kind: model.NodeKindFunction, Name: "auth", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}) + createAnnotation(t, db, strongScan.ID, "auth", model.DocTag{Kind: model.TagIntent, Value: "auth"}, model.DocTag{Kind: model.TagDomainRule, Value: "auth"}) + service := retrieval.Service{DB: db, SearchBackend: &stubSearchBackend{nodes: []model.Node{weakFTS}}} + + response, err := service.FromDB(context.Background(), "default", "auth", 2, 0, nil) + if err != nil { + t.Fatalf("FromDB returned error: %v", err) + } + if len(response.Results) != 2 { + t.Fatalf("expected FTS hit plus scan supplement, got %d: %+v", len(response.Results), response.Results) + } + if response.Results[0].ID != "file:a.go" { + t.Fatalf("search-engine hit must rank above a higher-scored scan supplement, got %q first", response.Results[0].ID) + } + if response.Results[1].ID != "file:b.go" { + t.Fatalf("expected scan supplement second, got %q", response.Results[1].ID) + } +} + func TestServiceFromDB_limitAppliesToFileGroups(t *testing.T) { db := newRetrievalDB(t) nodeA := createNode(t, db, model.Node{Namespace: "default", QualifiedName: "pkg.A", Kind: model.NodeKindFunction, Name: "NeedleA", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}) diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go new file mode 100644 index 0000000..3c67173 --- /dev/null +++ b/internal/safepath/safepath.go @@ -0,0 +1,94 @@ +// @index safepath는 여러 핸들러가 공유하는 저수준 경로 안전 프리미티브를 단일화한다. +// 정책(prefix/Rel/walk 기반 containment)과 에러 문자열은 호출부에 남기고, 여기서는 +// 공통 메커니즘(심링크 거부 walk, canonical 정규화, Rel 기반 containment)만 제공한다. +package safepath + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// EnsureNoSymlinkInPath walks each path segment from root to relPath rejecting symlinks. +// @intent prevent symlink traversal from escaping a trusted root before any filesystem mutation. +// @param allowMissingLeaf when true, returns the joined path even when the leaf does not yet exist. +func EnsureNoSymlinkInPath(root, relPath string, allowMissingLeaf bool) (string, error) { + cleanRel := filepath.Clean(relPath) + if cleanRel == "." { + return root, nil + } + current := root + segments := strings.Split(cleanRel, string(filepath.Separator)) + for i, segment := range segments { + current = filepath.Join(current, segment) + info, err := os.Lstat(current) + if err != nil { + if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) && i == len(segments)-1 { + return current, nil + } + if allowMissingLeaf && errors.Is(err, fs.ErrNotExist) { + continue + } + return "", err + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("symlink paths are not allowed") + } + } + return current, nil +} + +// Canonical resolves path to an absolute, symlink-free, cleaned filesystem path. +// @intent normalize user-supplied paths before containment comparison to prevent symlink-based escapes. +// @param allowMissingLeaf when true, a non-existent leaf is tolerated by resolving the parent and +// appending the unresolved base; when false, a missing path is an error (strict existence). +func Canonical(path string, allowMissingLeaf bool) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + clean := filepath.Clean(abs) + real, err := filepath.EvalSymlinks(clean) + if err == nil { + return filepath.Clean(real), nil + } + if !allowMissingLeaf { + return "", err + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + parent := filepath.Dir(clean) + base := filepath.Base(clean) + parentReal, parentErr := filepath.EvalSymlinks(parent) + if parentErr != nil { + return "", err + } + return filepath.Join(parentReal, base), nil +} + +// IsWithinRoot reports whether target is the same as root or a descendant of it. +// @intent detect path traversal by checking the relative path does not escape upward. +// @requires root and target should already be canonicalized (see Canonical) for symlink-safe comparison. +func IsWithinRoot(root, target string) (bool, error) { + rel, err := filepath.Rel(root, target) + if err != nil { + return false, err + } + if rel == "." { + return true, nil + } + if rel == ".." { + return false, nil + } + if len(rel) >= 3 && rel[:3] == ".."+string(filepath.Separator) { + return false, nil + } + if filepath.IsAbs(rel) { + return false, nil + } + return true, nil +} diff --git a/internal/safepath/safepath_test.go b/internal/safepath/safepath_test.go new file mode 100644 index 0000000..1ecfbf5 --- /dev/null +++ b/internal/safepath/safepath_test.go @@ -0,0 +1,123 @@ +package safepath_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/tae2089/code-context-graph/internal/safepath" +) + +func TestEnsureNoSymlinkInPath_AllowsPlainNestedPath(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "a", "b"), 0o755); err != nil { + t.Fatal(err) + } + got, err := safepath.EnsureNoSymlinkInPath(root, filepath.Join("a", "b"), false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != filepath.Join(root, "a", "b") { + t.Fatalf("unexpected path %q", got) + } +} + +func TestEnsureNoSymlinkInPath_DotReturnsRoot(t *testing.T) { + root := t.TempDir() + got, err := safepath.EnsureNoSymlinkInPath(root, ".", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != root { + t.Fatalf("expected root, got %q", got) + } +} + +func TestEnsureNoSymlinkInPath_RejectsSymlinkSegment(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink semantics differ on windows") + } + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outside, "secret"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + _, err := safepath.EnsureNoSymlinkInPath(root, filepath.Join("link", "secret"), false) + if err == nil { + t.Fatal("expected symlink segment to be rejected") + } +} + +func TestEnsureNoSymlinkInPath_MissingLeafToleratedWhenAllowed(t *testing.T) { + root := t.TempDir() + rel := filepath.Join("not", "there.txt") + got, err := safepath.EnsureNoSymlinkInPath(root, rel, true) + if err != nil { + t.Fatalf("expected missing leaf tolerated, got %v", err) + } + if got != filepath.Join(root, rel) { + t.Fatalf("unexpected path %q", got) + } +} + +func TestEnsureNoSymlinkInPath_MissingLeafErrorsWhenNotAllowed(t *testing.T) { + root := t.TempDir() + if _, err := safepath.EnsureNoSymlinkInPath(root, "missing.txt", false); err == nil { + t.Fatal("expected error for missing leaf when allowMissingLeaf is false") + } +} + +func TestCanonical_StrictRequiresExistence(t *testing.T) { + root := t.TempDir() + missing := filepath.Join(root, "nope") + if _, err := safepath.Canonical(missing, false); err == nil { + t.Fatal("expected strict Canonical to error on missing path") + } + got, err := safepath.Canonical(root, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + real, _ := filepath.EvalSymlinks(root) + if got != real { + t.Fatalf("expected canonical %q, got %q", real, got) + } +} + +func TestCanonical_TolerantResolvesParentForMissingLeaf(t *testing.T) { + root := t.TempDir() + real, _ := filepath.EvalSymlinks(root) + got, err := safepath.Canonical(filepath.Join(root, "newleaf"), true) + if err != nil { + t.Fatalf("expected tolerant Canonical to succeed, got %v", err) + } + if got != filepath.Join(real, "newleaf") { + t.Fatalf("unexpected path %q", got) + } +} + +func TestIsWithinRoot(t *testing.T) { + root := "/srv/data" + cases := []struct { + target string + want bool + }{ + {"/srv/data", true}, + {"/srv/data/sub/file", true}, + {"/srv/dataother", false}, + {"/srv", false}, + {"/etc/passwd", false}, + } + for _, c := range cases { + got, err := safepath.IsWithinRoot(root, c.target) + if err != nil { + t.Fatalf("IsWithinRoot(%q,%q) error: %v", root, c.target, err) + } + if got != c.want { + t.Errorf("IsWithinRoot(%q,%q)=%v want %v", root, c.target, got, c.want) + } + } +} diff --git a/internal/server/serve.go b/internal/server/serve.go index a68a2e7..d870731 100644 --- a/internal/server/serve.go +++ b/internal/server/serve.go @@ -112,9 +112,11 @@ func RunStreamableHTTP(rt *core.Runtime, srv *mcpgo.MCPServer, cfg Config, cache } return nil })) - mux.Handle("/status", StatusHandler(dbReadyCheck, cfg.WebhookAttemptTimeout, func() *webhook.SyncQueue { + // /status exposes repo names, branches, and raw error strings from the sync queue, + // so it requires the same bearer auth as /mcp; /health and /ready stay open for probes. + mux.Handle("/status", MCPAuthMiddleware(cfg.HTTPBearerToken, StatusHandler(dbReadyCheck, cfg.WebhookAttemptTimeout, func() *webhook.SyncQueue { return syncQueue - }, postprocessSummary)) + }, postprocessSummary))) if cfg.WikiDir != "" { wiki, err := wikiserver.New(wikiserver.Config{ diff --git a/internal/service/indexer.go b/internal/service/indexer.go index 6c4c6da..8340af9 100644 --- a/internal/service/indexer.go +++ b/internal/service/indexer.go @@ -135,12 +135,6 @@ func newParsedBuildNodeBatch(relPath string, content []byte, nodes []model.Node, return out } -// newParsedBuildEdgeBatch wraps parsed edges so they can be persisted after their owning nodes. -// @intent defer edge upserts until referenced nodes exist in the graph store. -func newParsedBuildEdgeBatch(relPath string, edges []model.Edge) parsedBuildEdgeBatch { - return parsedBuildEdgeBatch{relPath: relPath, edges: edges} -} - // bindAndReleaseNodeBatch upserts a parsed file's nodes and binds its comment annotations within a transaction-scoped store. // @intent persist nodes and their annotation bindings atomically per file before releasing comment buffers. // @sideEffect writes graph nodes and annotation rows via the transaction-scoped store. @@ -1210,8 +1204,10 @@ func (s *GraphService) updateGraphWithoutTx(ctx context.Context, absDir string, } stats := &incremental.SyncStats{} - normalExistingFiles := existingFilesExcluding(existingFiles, forceFiles) - passedExistingFiles := false + // Normal batches must never carry existingFiles: SyncWithExisting deletes existing files + // absent from the batch, so a multi-record spool would delete files belonging to later + // batches (then re-add them, churning node IDs and stats). Deletions are handled once, + // explicitly, below — mirroring the transactional path (applyUpdateSpoolInTx). for _, path := range spool.records { record, err := spool.readRecord(path) if err != nil { @@ -1221,19 +1217,14 @@ func (s *GraphService) updateGraphWithoutTx(ctx context.Context, absDir string, if len(normalFiles) == 0 { continue } - batchExistingFiles := []string(nil) - if !passedExistingFiles { - batchExistingFiles = normalExistingFiles - passedExistingFiles = true - } - batchStats, err := opts.Syncer.SyncWithExisting(ctx, normalFiles, batchExistingFiles) + batchStats, err := opts.Syncer.SyncWithExisting(ctx, normalFiles, nil) if err != nil { return nil, trace.Wrap(err, "incremental sync") } addSyncStats(stats, batchStats) } deletedFiles := existingFilesMissingFromSet(spool.currentFiles, existingFiles) - if len(deletedFiles) > 0 && passedExistingFiles { + if len(deletedFiles) > 0 { batchStats, err := opts.Syncer.SyncWithExisting(ctx, nil, deletedFiles) if err != nil { return nil, trace.Wrap(err, "incremental delete sync") @@ -1290,18 +1281,6 @@ func affectedUpdateFiles(currentHashes map[string]string, existingNodesByFile ma return files } -// existingFilesMissingFrom returns paths that were previously stored but are no longer present on disk. -// @intent identify deletions so the incremental syncer can remove their nodes and edges. -func existingFilesMissingFrom(files map[string]incremental.FileInfo, existingFiles []string) []string { - deleted := make([]string, 0) - for _, fp := range existingFiles { - if _, ok := files[fp]; !ok { - deleted = append(deleted, fp) - } - } - return deleted -} - // @intent detect deleted paths from the spool snapshot without rebuilding a full in-memory FileInfo map. func existingFilesMissingFromSet(currentFiles map[string]struct{}, existingFiles []string) []string { deleted := make([]string, 0) @@ -1313,25 +1292,6 @@ func existingFilesMissingFromSet(currentFiles map[string]struct{}, existingFiles return deleted } -// existingFilesExcluding filters out paths excluded from the current sync pass. -// @intent keep incremental sync from revisiting files the caller already ruled out. -func existingFilesExcluding(existingFiles []string, exclude map[string]struct{}) []string { - if len(existingFiles) == 0 { - return nil - } - if len(exclude) == 0 { - return append([]string(nil), existingFiles...) - } - filtered := make([]string, 0, len(existingFiles)) - for _, fp := range existingFiles { - if _, ok := exclude[fp]; ok { - continue - } - filtered = append(filtered, fp) - } - return filtered -} - // affectedNodeIDsForUpdate collects node IDs whose search documents must be refreshed for a given change set. // @intent merge previously stored node IDs with newly created ones so the search index sees both removals and additions. func affectedNodeIDsForUpdate(ctx context.Context, db *gorm.DB, existingNodesByFile map[string][]model.Node, changedFiles, deletedFiles []string) ([]uint, error) { @@ -1441,14 +1401,6 @@ func (s *GraphService) parserForExt(ext string) (Parser, bool) { return parser, ok } -// @intent rebuild the namespace-scoped search index after graph mutations when a search backend is configured. -func (s *GraphService) rebuildSearch(ctx context.Context) error { - if s.SearchBackend == nil || s.DB == nil { - return nil - } - return s.rebuildSearchWithDB(ctx, s.DB) -} - // rebuildSearchNodes refreshes search documents for the given node IDs when a search backend is configured. // @intent perform incremental search index updates after changed-node sets are known. func (s *GraphService) rebuildSearchNodes(ctx context.Context, nodeIDs []uint) error { @@ -1654,21 +1606,6 @@ func mergeLanguagePackages(dst map[string]languagePackageInfo, ambiguous map[str } } -// importPackageNames extracts a mapping of import paths to package names. -// @intent allow the edge resolver to look up package names by their source import paths. -func importPackageNames(packages map[string]languagePackageInfo) map[string]string { - if len(packages) == 0 { - return nil - } - names := make(map[string]string, len(packages)) - for importPath, pkg := range packages { - if importPath != "" && pkg.Name != "" { - names[importPath] = pkg.Name - } - } - return names -} - // @intent normalize discovered package imports into the canonical names used when resolving cross-file imports during parsing. func importPackageContext(packages map[string]languagePackageInfo) map[string]string { if len(packages) == 0 { diff --git a/internal/service/indexer_test.go b/internal/service/indexer_test.go index 48e0d04..ce85c8b 100644 --- a/internal/service/indexer_test.go +++ b/internal/service/indexer_test.go @@ -2701,6 +2701,40 @@ func TestUpdate_MaxTotalParsedBytesRejectsBeforeSync(t *testing.T) { } } +func TestUpdateGraphWithoutTx_DeletesEvenWithNoNormalBatches(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + if err != nil { + t.Fatalf("open db: %v", err) + } + if err := db.AutoMigrate(&model.Node{}); err != nil { + t.Fatalf("migrate nodes: %v", err) + } + if err := db.Create(&model.Node{Namespace: ctxns.DefaultNamespace, FilePath: "gone.go"}).Error; err != nil { + t.Fatalf("seed node: %v", err) + } + + svc := &GraphService{ + DB: db, + Walkers: map[string]*treesitter.Walker{".go": treesitter.NewWalker(treesitter.GoSpec)}, + Logger: slog.Default(), + } + + // Empty working dir: nothing to parse, so there are no normal batches. The previously + // stored gone.go must still be deleted (regression: the delete pass was gated on a normal + // batch having run). + tmpDir := t.TempDir() + syncer := &recordingIncrementalSyncer{result: &incremental.SyncStats{}} + if _, err := svc.Update(context.Background(), UpdateOptions{BuildOptions: BuildOptions{Dir: tmpDir, SkipSearchRebuild: true}, Syncer: syncer}); err != nil { + t.Fatalf("Update: %v", err) + } + if len(syncer.calls) != 1 { + t.Fatalf("expected a single delete pass, got %d calls: %v", len(syncer.calls), syncer.calls) + } + if got, want := syncer.calls[0].existingFiles, []string{"gone.go"}; !reflect.DeepEqual(got, want) { + t.Fatalf("delete pass existingFiles = %v, want %v", got, want) + } +} + func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) { db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { @@ -2709,6 +2743,8 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if err := db.AutoMigrate(&model.Node{}); err != nil { t.Fatalf("migrate nodes: %v", err) } + // Both seeded files are missing from disk. handler.go is inside the include path and must be + // deleted; helper.go is outside it and must be scoped out of the existing-file set entirely. if err := db.Create(&model.Node{Namespace: ctxns.DefaultNamespace, FilePath: filepath.Join("src", "api", "handler.go")}).Error; err != nil { t.Fatalf("seed api node: %v", err) } @@ -2727,8 +2763,10 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if err := os.MkdirAll(apiDir, 0o755); err != nil { t.Fatalf("mkdir api: %v", err) } - if err := os.WriteFile(filepath.Join(apiDir, "handler.go"), []byte("package api\n\nfunc Handler() {}\n"), 0o644); err != nil { - t.Fatalf("write handler: %v", err) + // A new file under the include path so the update has something to parse; the seeded + // handler.go is absent from disk and should be detected as a deletion. + if err := os.WriteFile(filepath.Join(apiDir, "new.go"), []byte("package api\n\nfunc New() {}\n"), 0o644); err != nil { + t.Fatalf("write new: %v", err) } syncer := &recordingIncrementalSyncer{result: &incremental.SyncStats{}} @@ -2736,8 +2774,18 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if err != nil { t.Fatalf("Update: %v", err) } - if got, want := syncer.existingFiles, []string{filepath.Join("src", "api", "handler.go")}; !reflect.DeepEqual(got, want) { - t.Fatalf("existingFiles mismatch: got=%v want=%v", got, want) + + var deleteCall *recordingSyncCall + for i := range syncer.calls { + if len(syncer.calls[i].files) == 0 && len(syncer.calls[i].existingFiles) > 0 { + deleteCall = &syncer.calls[i] + } + } + if deleteCall == nil { + t.Fatalf("expected a delete pass for the missing include-path file, calls=%v", syncer.calls) + } + if got, want := deleteCall.existingFiles, []string{filepath.Join("src", "api", "handler.go")}; !reflect.DeepEqual(got, want) { + t.Fatalf("delete pass existingFiles mismatch: got=%v want=%v (helper.go outside include path must be scoped out)", got, want) } } @@ -3045,11 +3093,10 @@ func TestUpdateGraphWithoutTx_DoesNotDeleteForcedFilesDuringNormalSync(t *testin if _, ok := first.files["target.go"]; !ok { t.Fatalf("expected changed target.go in normal sync files, got %v", sortedIncrementalFileKeys(first.files)) } - if slices.Contains(first.existingFiles, "source.go") { - t.Fatalf("expected forced source.go to be excluded from normal sync existingFiles, got %v", first.existingFiles) - } - if !slices.Contains(first.existingFiles, "deleted.go") { - t.Fatalf("expected deleted.go to remain in normal sync existingFiles, got %v", first.existingFiles) + // Normal sync passes no existingFiles: deletions are handled by the separate delete pass, + // so a normal batch can never delete files that belong to other batches (mirrors the tx path). + if len(first.existingFiles) != 0 { + t.Fatalf("expected normal sync to pass no existingFiles, got %v", first.existingFiles) } second := syncer.calls[1] if len(second.files) != 0 { diff --git a/internal/store/gormstore/gormstore.go b/internal/store/gormstore/gormstore.go index c925508..a46d23f 100644 --- a/internal/store/gormstore/gormstore.go +++ b/internal/store/gormstore/gormstore.go @@ -4,6 +4,7 @@ package gormstore import ( "context" "errors" + "fmt" "path" "strings" @@ -516,6 +517,17 @@ func (s *Store) UpsertAnnotation(ctx context.Context, ann *model.Annotation) err if errors.Is(result.Error, gorm.ErrRecordNotFound) { return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // The read path is namespace-scoped; the create path must be too, or a caller + // could attach an annotation to another namespace's node (or a missing node). + var owned int64 + if err := tx.Model(&model.Node{}). + Where("id = ? AND namespace = ?", ann.NodeID, ns). + Count(&owned).Error; err != nil { + return trace.Wrap(err, "verify annotation node namespace") + } + if owned == 0 { + return fmt.Errorf("annotation node %d not found in namespace %q", ann.NodeID, ns) + } return tx.Create(ann).Error }) } diff --git a/internal/store/gormstore/gormstore_test.go b/internal/store/gormstore/gormstore_test.go index b172f71..30c151c 100644 --- a/internal/store/gormstore/gormstore_test.go +++ b/internal/store/gormstore/gormstore_test.go @@ -9,7 +9,7 @@ import ( "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" - gormschema "gorm.io/gorm/schema" + gormschema "gorm.io/gorm/schema" "github.com/tae2089/code-context-graph/internal/ctxns" "github.com/tae2089/code-context-graph/internal/model" @@ -626,6 +626,31 @@ func TestUpsertAnnotation_Insert(t *testing.T) { } } +func TestUpsertAnnotation_RejectsNodeOutsideNamespace(t *testing.T) { + s := setupTestDB(t) + ctx := context.Background() + + s.UpsertNodes(ctx, []model.Node{ + {QualifiedName: "pkg.F", Kind: model.NodeKindFunction, Name: "F", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}, + }) + node, _ := s.GetNode(ctx, "pkg.F") + + // Writing from another namespace must not create an annotation on a foreign node. + foreignCtx := ctxns.WithNamespace(context.Background(), "other-team") + err := s.UpsertAnnotation(foreignCtx, &model.Annotation{NodeID: node.ID, Summary: "cross-namespace write"}) + if err == nil { + t.Fatal("expected cross-namespace annotation create to be rejected") + } + if got, _ := s.GetAnnotation(ctx, node.ID); got != nil { + t.Fatalf("annotation must not exist after rejected write, got %+v", got) + } + + // Nonexistent node id must also be rejected instead of creating an orphan row. + if err := s.UpsertAnnotation(ctx, &model.Annotation{NodeID: 99999, Summary: "orphan"}); err == nil { + t.Fatal("expected annotation create for missing node to be rejected") + } +} + func TestUpsertAnnotation_Update(t *testing.T) { s := setupTestDB(t) ctx := context.Background() diff --git a/internal/store/search/postgres.go b/internal/store/search/postgres.go index 3b0404c..b0c9a24 100644 --- a/internal/store/search/postgres.go +++ b/internal/store/search/postgres.go @@ -4,6 +4,8 @@ package search import ( "context" "fmt" + "log/slog" + "strconv" "gorm.io/gorm" @@ -13,6 +15,11 @@ import ( "github.com/tae2089/code-context-graph/internal/model" ) +// fuzzyWordSimilarityThreshold is the minimum pg_trgm word_similarity for a symbol name to be +// accepted as a typo-tolerant fuzzy match. Tuned to admit single-character typos while rejecting +// unrelated names. +const fuzzyWordSimilarityThreshold = 0.4 + // PostgresBackend is a full-text search backend based on PostgreSQL tsvector. // @intent Handles full-text search indexing and querying in a PostgreSQL environment. type PostgresBackend struct{} @@ -74,6 +81,20 @@ func (p *PostgresBackend) Migrate(db *gorm.DB) error { return trace.Wrap(err, "create gin index") } + // pg_trgm powers typo-tolerant fuzzy symbol matching. The extension may need + // elevated privileges, so treat its absence as a graceful downgrade to exact FTS + // rather than a fatal migration error. + if err := db.Exec(`CREATE EXTENSION IF NOT EXISTS pg_trgm`).Error; err != nil { + slog.Warn("pg_trgm unavailable; fuzzy symbol search disabled", trace.SlogError(err)) + return nil + } + if err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_name_trgm ON nodes USING gin (name gin_trgm_ops)`).Error; err != nil { + return trace.Wrap(err, "create name trgm index") + } + if err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_qualified_name_trgm ON nodes USING gin (qualified_name gin_trgm_ops)`).Error; err != nil { + return trace.Wrap(err, "create qualified_name trgm index") + } + return nil } @@ -154,8 +175,12 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string, return nil, trace.Wrap(err, "ts_query") } + // A zero-hit tsquery is not terminal: a misspelled query legitimately matches nothing + // exactly, and the pg_trgm fuzzy supplement can still surface candidates. Fuzzy fires + // only on a total exact miss so it never dilutes the precision of queries that did match. if len(rows) == 0 { - return nil, nil + fuzzy := p.appendFuzzyMatches(ctx, db, query, ns, limit) + return promoteExactNameMatch(fuzzy, query), nil } nodeIDs := make([]uint, len(rows)) @@ -191,4 +216,70 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string, return result, nil } +// appendFuzzyMatches returns up to limit pg_trgm fuzzy matches on symbol names for queries +// that produced no exact FTS hit (typo tolerance). It is best-effort: if pg_trgm is +// unavailable the query errors and an empty result is returned so exact search still works. +// @intent add typo tolerance to Postgres search without letting a missing extension break exact search. +// @domainRule only nodes that also have a search_documents row are eligible, so fuzzy stays +// within the same corpus and kind mix as the exact FTS path. +func (p *PostgresBackend) appendFuzzyMatches(ctx context.Context, db *gorm.DB, query, ns string, limit int) []model.Node { + if limit <= 0 { + return nil + } + + // The `<%` operator is index-accelerated by the gin_trgm_ops indexes on name/qualified_name + // (a functional `word_similarity(...) >= const` filter cannot use them); its cutoff is the + // session-local word_similarity_threshold, set here so the operator and our tuned threshold + // agree. Ordering uses the functional form over the already-filtered small set. + var rows []resultRow + fuzzySQL := ` + SELECT n.id AS node_id + FROM nodes n + JOIN search_documents sd ON sd.node_id = n.id AND sd.namespace = n.namespace + WHERE n.namespace = ? + AND (? <% n.name OR ? <% n.qualified_name) + ORDER BY GREATEST(word_similarity(?, n.name), word_similarity(?, n.qualified_name)) DESC + LIMIT ?` + err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if e := tx.Exec(`SELECT set_config('pg_trgm.word_similarity_threshold', ?, true)`, + strconv.FormatFloat(fuzzyWordSimilarityThreshold, 'f', -1, 64)).Error; e != nil { + return e + } + return tx.Raw(fuzzySQL, ns, query, query, query, query, limit).Scan(&rows).Error + }) + if err != nil { + // Context cancellation is the caller's concern, not a fuzzy fault; only surface real errors. + if ctx.Err() == nil { + slog.Warn("pg_trgm fuzzy supplement failed", trace.SlogError(err)) + } + return nil + } + if len(rows) == 0 { + return nil + } + + nodeIDs := make([]uint, len(rows)) + for i, r := range rows { + nodeIDs[i] = r.NodeID + } + var nodes []model.Node + if err := db.WithContext(ctx).Where("id IN ?", nodeIDs).Where("namespace = ?", ns).Find(&nodes).Error; err != nil { + if ctx.Err() == nil { + slog.Warn("pg_trgm fuzzy node load failed", trace.SlogError(err)) + } + return nil + } + byID := make(map[uint]model.Node, len(nodes)) + for _, n := range nodes { + byID[n.ID] = n + } + ordered := make([]model.Node, 0, len(nodeIDs)) + for _, id := range nodeIDs { + if n, ok := byID[id]; ok && n.ID != 0 { + ordered = append(ordered, n) + } + } + return ordered +} + var _ Backend = (*PostgresBackend)(nil) diff --git a/internal/store/search/postgres_test.go b/internal/store/search/postgres_test.go index 6caa22a..e66ac4b 100644 --- a/internal/store/search/postgres_test.go +++ b/internal/store/search/postgres_test.go @@ -133,6 +133,15 @@ func TestPostgresFTS_Rebuild(t *testing.T) { } } +// disableTSVTrigger turns off the tsv BEFORE INSERT/UPDATE trigger so tests can stage +// stale tsv values by hand. Tables are dropped per test, so re-enabling is unnecessary. +func disableTSVTrigger(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec("ALTER TABLE search_documents DISABLE TRIGGER trg_search_documents_tsv").Error; err != nil { + t.Fatalf("disable tsv trigger: %v", err) + } +} + func TestPostgresFTS_RebuildNodes_RefreshesOnlyScopedRows(t *testing.T) { db := setupPostgresDB(t) backend := NewPostgresBackend() @@ -157,6 +166,10 @@ func TestPostgresFTS_RebuildNodes_RefreshesOnlyScopedRows(t *testing.T) { if err := db.Create(&model.SearchDocument{Namespace: "ns-b", NodeID: foreign.ID, Content: "fresh foreign", Language: "go"}).Error; err != nil { t.Fatal(err) } + // The tsv BEFORE UPDATE trigger would recompute tsv from content and overwrite the + // manual stale marker; disable it so the stale state actually exists and RebuildNodes' + // own UPDATE statement is what gets verified. + disableTSVTrigger(t, db) if err := db.Exec("UPDATE search_documents SET tsv = to_tsvector('simple', 'stale')").Error; err != nil { t.Fatal(err) } @@ -193,6 +206,9 @@ func TestPostgresFTS_RebuildNodes_EmptyScopeIsNoOp(t *testing.T) { if err := db.Create(&model.SearchDocument{Namespace: ctxns.DefaultNamespace, NodeID: node.ID, Content: "fresh keep", Language: "go"}).Error; err != nil { t.Fatal(err) } + // See TestPostgresFTS_RebuildNodes_RefreshesOnlyScopedRows: the trigger must be off + // for the stale marker to persist. + disableTSVTrigger(t, db) if err := db.Exec("UPDATE search_documents SET tsv = to_tsvector('simple', 'stale')").Error; err != nil { t.Fatal(err) } @@ -275,6 +291,35 @@ func TestPostgresFTS_Query(t *testing.T) { } } +func TestPostgresFTS_Query_FuzzyTypoFallsBackToTrigram(t *testing.T) { + db := setupPostgresDB(t) + seedPostgresNodes(t, db) + + backend := NewPostgresBackend() + if err := backend.Migrate(db); err != nil { + t.Fatal(err) + } + if err := backend.Rebuild(context.Background(), db); err != nil { + t.Fatal(err) + } + + // "authentcate" is a typo (missing 'i') that exact tsquery cannot match; + // the pg_trgm supplement should still surface AuthenticateUser by symbol-name similarity. + nodes, err := backend.Query(context.Background(), db, "authentcate", 10) + if err != nil { + t.Fatal(err) + } + found := false + for _, n := range nodes { + if n.QualifiedName == "pkg.AuthenticateUser" { + found = true + } + } + if !found { + t.Errorf("expected fuzzy trigram match to surface pkg.AuthenticateUser for typo query, got %+v", nodes) + } +} + func TestPostgresFTS_Query_NamespaceIsolation(t *testing.T) { db := setupPostgresDB(t) backend := NewPostgresBackend() diff --git a/internal/webhook/syncqueue_test.go b/internal/webhook/syncqueue_test.go index 89018c3..7ee818c 100644 --- a/internal/webhook/syncqueue_test.go +++ b/internal/webhook/syncqueue_test.go @@ -14,35 +14,44 @@ import ( ) func TestSyncQueue_DeduplicatesRapidPushes(t *testing.T) { - var callCount atomic.Int32 - done := make(chan struct{}) + var svcCalls atomic.Int32 + gate := make(chan struct{}) + svcDone := make(chan struct{}) handler := func(_ context.Context, repoFullName, cloneURL, branch string) error { - callCount.Add(1) - time.Sleep(50 * time.Millisecond) - if callCount.Load() == 1 { - close(done) + switch repoFullName { + case "org/blocker": + <-gate + case "org/svc": + if svcCalls.Add(1) == 1 { + close(svcDone) + } } return nil } - q := NewSyncQueue(2, handler) + // A single worker held by the gated blocker guarantees all three pushes arrive while + // org/svc is queued (never processing), so dedup must collapse them into one run. + // Racing the pushes against a free worker made this test flaky under parallel load. + q := NewSyncQueue(1, handler) defer q.Shutdown() + q.Add(context.Background(), "org/blocker", "https://github.com/org/blocker.git", "main") q.Add(context.Background(), "org/svc", "https://github.com/org/svc.git", "main") q.Add(context.Background(), "org/svc", "https://github.com/org/svc.git", "main") q.Add(context.Background(), "org/svc", "https://github.com/org/svc.git", "main") + close(gate) select { - case <-done: + case <-svcDone: case <-time.After(5 * time.Second): t.Fatal("timed out waiting for handler") } + // Grace window so an erroneous duplicate run would surface before the assertion. time.Sleep(100 * time.Millisecond) - got := callCount.Load() - if got != 1 { + if got := svcCalls.Load(); got != 1 { t.Errorf("handler called %d times, want 1 (dedup failed)", got) } } diff --git a/internal/wikiserver/server.go b/internal/wikiserver/server.go index 28cbb35..8849b09 100644 --- a/internal/wikiserver/server.go +++ b/internal/wikiserver/server.go @@ -552,13 +552,6 @@ type contextResponse struct { Items []contextItem `json:"items"` } -// @intent return retrieval metadata and optional bounded Markdown content to the Wiki UI. -type retrieveResult struct { - ragindex.RetrieveResult - Content string `json:"content,omitempty"` - ContentTruncated bool `json:"content_truncated,omitempty"` -} - // @intent return the resolved Wiki navigation target for one ccg:// ref. type refResponse struct { Namespace string `json:"namespace"` @@ -723,6 +716,8 @@ func readDocFile(resolved string) (string, string, error) { } // @intent resolve a generated doc path under approved docs, RAG, or namespace roots. +// @domainRule the working directory is only searched through its docs/ subtree; arbitrary +// repository files (config, source, secrets) must never be readable through the doc API. func (s *Server) resolveDocPath(namespace, docPath string) (string, error) { clean := filepath.Clean(docPath) if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { @@ -732,7 +727,9 @@ func (s *Server) resolveDocPath(namespace, docPath string) (string, error) { if namespace != ctxns.DefaultNamespace { roots = append(roots, filepath.Join(s.namespaceRoot, namespace)) } - roots = append(roots, ".", s.ragIndexDir, s.namespaceRoot) + // Generated shared docs live under ./docs (doc_path values carry the docs/ prefix), + // so the local root is the docs directory itself, not the bare working directory. + roots = append(roots, "docs", s.ragIndexDir, s.namespaceRoot) if filepath.IsAbs(clean) { for _, root := range roots { target, err := safeAbsolutePath(root, clean) @@ -746,7 +743,17 @@ func (s *Server) resolveDocPath(namespace, docPath string) (string, error) { return "", fs.ErrNotExist } for _, root := range roots { - target, err := safePath(root, clean) + rel := clean + if root == "docs" { + // doc_path values are docs-prefixed ("docs/pkg/file.md"); resolve the remainder + // inside the docs root so containment is enforced against docs/, not the CWD. + after, ok := strings.CutPrefix(clean, "docs"+string(os.PathSeparator)) + if !ok { + continue + } + rel = after + } + target, err := safePath(root, rel) if err != nil { continue } diff --git a/internal/wikiserver/server_test.go b/internal/wikiserver/server_test.go index 48fbf02..9a4b0d5 100644 --- a/internal/wikiserver/server_test.go +++ b/internal/wikiserver/server_test.go @@ -690,6 +690,32 @@ func TestAPI_DocReturnsNotFoundWithoutDBFallback(t *testing.T) { } } +func TestAPI_DocRejectsNonDocsFilesUnderCwd(t *testing.T) { + srv := newTestServer(t) + // A sensitive non-docs file in the working directory must not be readable as a "doc". + if err := os.WriteFile("secret.env", []byte("TOKEN=hunter2"), 0o644); err != nil { + t.Fatalf("write secret: %v", err) + } + for _, p := range []string{"secret.env", "./secret.env"} { + req := httptest.NewRequest(http.MethodGet, "/wiki/api/doc?namespace=default&path="+p, nil) + rec := httptest.NewRecorder() + srv.APIHandler().ServeHTTP(rec, req) + if rec.Code == http.StatusOK { + t.Fatalf("path %q: non-docs CWD file served as doc: %s", p, rec.Body.String()) + } + } + abs, err := filepath.Abs("secret.env") + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodGet, "/wiki/api/doc?namespace=default&path="+abs, nil) + rec := httptest.NewRecorder() + srv.APIHandler().ServeHTTP(rec, req) + if rec.Code == http.StatusOK { + t.Fatalf("absolute path: non-docs CWD file served as doc: %s", rec.Body.String()) + } +} + func TestAPI_DocRejectsTraversal(t *testing.T) { srv := newTestServer(t) req := httptest.NewRequest(http.MethodGet, "/wiki/api/doc?namespace=default&path=../secret.md", nil) diff --git a/scripts/eval.sh b/scripts/eval.sh deleted file mode 100755 index 42235a6..0000000 --- a/scripts/eval.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -CCG_BIN=${CCG_BIN:-ccg} -CCG_EVAL_CORPUS=${CCG_EVAL_CORPUS:-testdata/eval} -CCG_EVAL_NAMESPACE=${CCG_EVAL_NAMESPACE:-eval} -CCG_EVAL_DB_DRIVER=${CCG_EVAL_DB_DRIVER:-sqlite} -CCG_EVAL_DB_DSN=${CCG_EVAL_DB_DSN:-} -CCG_EVAL_KEEP_DB=${CCG_EVAL_KEEP_DB:-0} - -eval_default_db_path() { - local tmpdir - tmpdir=$(mktemp -d) - printf '%s/eval.db\n' "$tmpdir" -} - -eval_db_dsn() { - if [ -n "$CCG_EVAL_DB_DSN" ]; then - printf '%s\n' "$CCG_EVAL_DB_DSN" - return 0 - fi - eval_default_db_path -} - -eval_build_cmd() { - local db="$1" - printf '%s\n' "$CCG_BIN build $CCG_EVAL_CORPUS --db-driver $CCG_EVAL_DB_DRIVER --db-dsn $db --namespace $CCG_EVAL_NAMESPACE" -} - -eval_migrate_cmd() { - local db="$1" - printf '%s\n' "$CCG_BIN migrate --db-driver $CCG_EVAL_DB_DRIVER --db-dsn $db" -} - -eval_run_cmd() { - local db="$1" - printf '%s\n' "$CCG_BIN eval --corpus $CCG_EVAL_CORPUS --suite all --db-driver $CCG_EVAL_DB_DRIVER --db-dsn $db --namespace $CCG_EVAL_NAMESPACE" -} - -run_eval_build() { - local db="$1" - # shellcheck disable=SC2086 - $CCG_BIN build "$CCG_EVAL_CORPUS" \ - --db-driver "$CCG_EVAL_DB_DRIVER" \ - --db-dsn "$db" \ - --namespace "$CCG_EVAL_NAMESPACE" -} - -run_eval_migrate() { - local db="$1" - # shellcheck disable=SC2086 - $CCG_BIN migrate \ - --db-driver "$CCG_EVAL_DB_DRIVER" \ - --db-dsn "$db" -} - -run_eval() { - local db="$1" - # shellcheck disable=SC2086 - $CCG_BIN eval \ - --corpus "$CCG_EVAL_CORPUS" \ - --suite all \ - --db-driver "$CCG_EVAL_DB_DRIVER" \ - --db-dsn "$db" \ - --namespace "$CCG_EVAL_NAMESPACE" -} - -cleanup_eval_db() { - local db="$1" - if [ "$CCG_EVAL_KEEP_DB" = "1" ]; then - return 0 - fi - rm -f "$db" - rmdir "$(dirname "$db")" 2>/dev/null || true -} - -main() { - local db - db=$(eval_db_dsn) - trap "cleanup_eval_db '$db'" EXIT - run_eval_migrate "$db" - run_eval_build "$db" - run_eval "$db" -} - -if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then - main "$@" -fi diff --git a/scripts/eval_test.sh b/scripts/eval_test.sh deleted file mode 100755 index 8756e27..0000000 --- a/scripts/eval_test.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -ROOT_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) - -source "${ROOT_DIR}/scripts/eval.sh" - -assert_equals() { - local expected="$1" actual="$2" - [ "$expected" = "$actual" ] || { - echo "expected '$expected', got '$actual'" >&2 - exit 1 - } -} - -assert_contains() { - local haystack="$1" needle="$2" - [[ "$haystack" == *"$needle"* ]] || { - echo "expected '$haystack' to contain '$needle'" >&2 - exit 1 - } -} - -assert_fails() { - local description="$1"; shift - if ( "$@" ) >/dev/null 2>&1; then - echo "expected failure: ${description}" >&2 - exit 1 - fi -} - -reset_test_env() { - CCG_BIN="ccg" - CCG_EVAL_CORPUS="testdata/eval" - CCG_EVAL_NAMESPACE="eval" - CCG_EVAL_DB_DRIVER="sqlite" - CCG_EVAL_DB_DSN="" - CCG_EVAL_KEEP_DB=0 - TEST_TMPDIR="" -} - -test_eval_script_exists_and_is_executable() { - [ -x "${ROOT_DIR}/scripts/eval.sh" ] || { - echo "expected scripts/eval.sh to exist and be executable" >&2 - exit 1 - } -} - -test_eval_default_db_path_uses_temp_dir() { - local path - path=$(eval_default_db_path) - assert_contains "$path" "/eval.db" - [ -d "$(dirname "$path")" ] || { - echo "expected temp dir to exist for path: $path" >&2 - exit 1 - } - rm -rf "$(dirname "$path")" -} - -test_eval_build_and_run_cmds_use_shared_db_namespace_and_corpus() { - local db migrate_cmd build_cmd run_cmd - db="/tmp/eval-script-test.db" - migrate_cmd=$(eval_migrate_cmd "$db") - build_cmd=$(eval_build_cmd "$db") - run_cmd=$(eval_run_cmd "$db") - - assert_equals "ccg migrate --db-driver sqlite --db-dsn /tmp/eval-script-test.db" "$migrate_cmd" - assert_equals "ccg build testdata/eval --db-driver sqlite --db-dsn /tmp/eval-script-test.db --namespace eval" "$build_cmd" - assert_equals "ccg eval --corpus testdata/eval --suite all --db-driver sqlite --db-dsn /tmp/eval-script-test.db --namespace eval" "$run_cmd" -} - -test_eval_main_invokes_build_then_eval_in_order() { - local tmpdir stub_dir log db - tmpdir=$(mktemp -d) - stub_dir="${tmpdir}/bin" - log="${tmpdir}/ccg.log" - db="${tmpdir}/shared.db" - mkdir -p "$stub_dir" - - cat > "${stub_dir}/ccg" <> "$log" -EOF - chmod +x "${stub_dir}/ccg" - - PATH="${stub_dir}:$PATH" CCG_EVAL_DB_DSN="$db" main >/dev/null - - [ -f "$log" ] || { - echo "expected stub log to be created" >&2 - exit 1 - } - - local line1 line2 line3 - line1=$(sed -n '1p' "$log") - line2=$(sed -n '2p' "$log") - line3=$(sed -n '3p' "$log") - assert_equals "migrate --db-driver sqlite --db-dsn $db" "$line1" - assert_equals "build testdata/eval --db-driver sqlite --db-dsn $db --namespace eval" "$line2" - assert_equals "eval --corpus testdata/eval --suite all --db-driver sqlite --db-dsn $db --namespace eval" "$line3" - - rm -rf "$tmpdir" -} - -test_eval_main_supports_multitoken_ccg_bin_override() { - local tmpdir stub_dir log db - tmpdir=$(mktemp -d) - stub_dir="${tmpdir}/bin" - log="${tmpdir}/ccg.log" - db="${tmpdir}/shared.db" - mkdir -p "$stub_dir" - - cat > "${stub_dir}/ccg-stub" <> "$log" -EOF - chmod +x "${stub_dir}/ccg-stub" - - PATH="${stub_dir}:$PATH" CCG_BIN='bash ccg-stub' CCG_EVAL_DB_DSN="$db" main >/dev/null - - local line1 line2 line3 - line1=$(sed -n '1p' "$log") - line2=$(sed -n '2p' "$log") - line3=$(sed -n '3p' "$log") - assert_equals "migrate --db-driver sqlite --db-dsn $db" "$line1" - assert_equals "build testdata/eval --db-driver sqlite --db-dsn $db --namespace eval" "$line2" - assert_equals "eval --corpus testdata/eval --suite all --db-driver sqlite --db-dsn $db --namespace eval" "$line3" - - rm -rf "$tmpdir" -} - -run_test() { - local name="$1" - if ( reset_test_env; "$name" ); then - printf 'ok - %s\n' "$name" - else - printf 'not ok - %s\n' "$name" >&2 - exit 1 - fi -} - -run_test test_eval_script_exists_and_is_executable -run_test test_eval_default_db_path_uses_temp_dir -run_test test_eval_build_and_run_cmds_use_shared_db_namespace_and_corpus -run_test test_eval_main_invokes_build_then_eval_in_order -run_test test_eval_main_supports_multitoken_ccg_bin_override - -echo "eval helper tests passed"