From c53c10505e8c75632aca4be32fe20e9e8d5abec2 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Tue, 31 Mar 2026 21:15:19 -0700 Subject: [PATCH 01/36] feat: add --coverage flag for code coverage tracking POC Adds code coverage collection during test execution: - --coverage flag on `tusk run` enables coverage mode - CLI injects NODE_V8_COVERAGE and TUSK_COVERAGE_PORT env vars - Takes V8 coverage snapshots between tests via SDK HTTP endpoint - Processes raw V8 files with embedded Node.js helper script - Runs c8 report for aggregate Istanbul JSON - Diffs consecutive V8 snapshots for per-test coverage (marginal/new lines) - Outputs per-file and per-test coverage summary - Shows coverage sub-lines in --print mode - Shows per-file coverage breakdown in TUI test log panel - Cleans up raw V8 files after processing (~40KB output vs ~24MB raw) Known limitation: per-test coverage shows marginal (new) lines only due to V8's binary best-effort coverage mode. Will be addressed by switching to V8 Inspector precise coverage with reset between tests. --- cmd/run.go | 90 +++ internal/runner/coverage.go | 560 ++++++++++++++++++ internal/runner/executor.go | 40 ++ .../runner/scripts/process-v8-coverage.js | 82 +++ internal/runner/service.go | 23 + internal/runner/types.go | 1 + internal/tui/test_executor.go | 25 + 7 files changed, 821 insertions(+) create mode 100644 internal/runner/coverage.go create mode 100644 internal/runner/scripts/process-v8-coverage.js diff --git a/cmd/run.go b/cmd/run.go index c9b5621..b2b46e1 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -55,6 +55,9 @@ var ( // Validation mode validateSuiteIfDefaultBranch bool validateSuite bool + + // Coverage mode + coverageEnabled bool ) //go:embed short_docs/drift/drift_run.md @@ -115,6 +118,9 @@ func bindRunFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&validateSuiteIfDefaultBranch, "validate-suite-if-default-branch", false, "[Cloud] Validate traces on default branch before adding to suite") cmd.Flags().BoolVar(&validateSuite, "validate-suite", false, "[Cloud] Force validation mode regardless of branch") + // Coverage mode + cmd.Flags().BoolVar(&coverageEnabled, "coverage", false, "Collect code coverage during test execution") + _ = cmd.Flags().MarkHidden("client-id") cmd.Flags().SortFlags = false } @@ -303,6 +309,10 @@ func runTests(cmd *cobra.Command, args []string) error { } executor.SetEnableServiceLogs(enableServiceLogs || debug) + if coverageEnabled { + executor.SetCoverageEnabled(true) + log.Stderrln("➀ Coverage collection enabled") + } if saveResults { if resultsDir == "" { if getConfigErr == nil && cfg.Results.Dir != "" { @@ -389,6 +399,54 @@ func runTests(cmd *cobra.Command, args []string) error { }) } + // Coverage: wrap the OnTestCompleted callback to take snapshots between tests + var coverageRecords []runner.CoverageTestRecord + var coverageMu sync.Mutex + if coverageEnabled { + existingCallback := executor.OnTestCompleted + executor.SetOnTestCompleted(func(res runner.TestResult, test runner.Test) { + // Call the original callback first + if existingCallback != nil { + existingCallback(res, test) + } + + // Take coverage snapshot and process immediately (extract line counts, clean up raw V8 file) + lineCounts, err := executor.TakeCoverageSnapshotAndProcess() + if err != nil { + log.Warn("Failed to take coverage snapshot", "testID", test.TraceID, "error", err) + return + } + + coverageMu.Lock() + // Compute per-test diff (delta from previous snapshot) + var prevCounts map[string]map[string]int + if len(coverageRecords) > 0 { + prevCounts = coverageRecords[len(coverageRecords)-1].LineCounts + } + coverageRecords = append(coverageRecords, runner.CoverageTestRecord{ + TestID: test.TraceID, + TestName: fmt.Sprintf("%s %s", test.Method, test.Path), + LineCounts: lineCounts, + }) + coverageMu.Unlock() + + // Diff with previous snapshot to get this test's coverage + diff := runner.DiffV8LineCounts(prevCounts, lineCounts) + executor.SetTestCoverageDetail(test.TraceID, diff) + + // Print sub-line in --print mode + if !interactive { + totalLines := 0 + for _, fd := range diff { + totalLines += fd.CoveredCount + } + if totalLines > 0 { + log.UserProgress(fmt.Sprintf(" ↳ coverage: %d lines across %d files", totalLines, len(diff))) + } + } + }) + } + var tests []runner.Test var err error @@ -822,6 +880,25 @@ func runTests(cmd *cobra.Command, args []string) error { log.Stderrln(fmt.Sprintf("➀ Running %d tests (concurrency: %d)...\n", len(tests), executor.GetConcurrency())) } + // Coverage: take baseline snapshot before any tests run + if coverageEnabled { + // Small delay to let the service fully initialize and coverage server start + time.Sleep(500 * time.Millisecond) + baselineCounts, err := executor.TakeCoverageSnapshotAndProcess() + if err != nil { + log.Warn("Failed to take baseline coverage snapshot", "error", err) + } else { + coverageMu.Lock() + coverageRecords = append(coverageRecords, runner.CoverageTestRecord{ + TestID: "_baseline", + TestName: "baseline", + LineCounts: baselineCounts, + }) + coverageMu.Unlock() + log.Debug("Coverage baseline snapshot taken") + } + } + results, err = executor.RunTests(tests) if err != nil { cmd.SilenceUsage = true @@ -852,6 +929,19 @@ func runTests(cmd *cobra.Command, args []string) error { _ = os.Stdout.Sync() time.Sleep(1 * time.Millisecond) + // Coverage: process coverage data after all tests complete (before service stops) + if coverageEnabled && len(coverageRecords) > 1 { + // records[0] = baseline, records[1..N] = per-test (LineCounts already processed inline) + coverageMu.Lock() + records := make([]runner.CoverageTestRecord, len(coverageRecords)) + copy(records, coverageRecords) + coverageMu.Unlock() + + if err := executor.ProcessCoverage(records); err != nil { + log.Warn("Failed to process coverage", "error", err) + } + } + var outputErr error if !interactive { // Results already streamed, just print summary diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go new file mode 100644 index 0000000..c753266 --- /dev/null +++ b/internal/runner/coverage.go @@ -0,0 +1,560 @@ +package runner + +import ( + _ "embed" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Use-Tusk/tusk-cli/internal/log" +) + +//go:embed scripts/process-v8-coverage.js +var processV8CoverageScript string + +const coverageSnapshotTimeout = 5 * time.Second + +// TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. +func (e *Executor) TakeCoverageSnapshot() error { + if !e.coverageEnabled || e.coveragePort == 0 { + return nil + } + + url := fmt.Sprintf("http://127.0.0.1:%d/snapshot", e.coveragePort) + client := &http.Client{Timeout: coverageSnapshotTimeout} + + resp, err := client.Get(url) + if err != nil { + return fmt.Errorf("failed to take coverage snapshot: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("coverage snapshot returned status %d: %s", resp.StatusCode, string(body)) + } + + // Small delay to let V8 finish writing the file + time.Sleep(100 * time.Millisecond) + return nil +} + +// TakeCoverageSnapshotAndProcess takes a snapshot, processes the latest V8 file +// into compact line counts, and deletes the raw V8 file to save disk space. +// Returns the processed line counts. +func (e *Executor) TakeCoverageSnapshotAndProcess() (map[string]map[string]int, error) { + if err := e.TakeCoverageSnapshot(); err != nil { + return nil, err + } + + files, err := e.ListV8CoverageFiles() + if err != nil { + return nil, err + } + if len(files) == 0 { + return make(map[string]map[string]int), nil + } + + // Process the latest file (cumulative snapshot) + latestFile := files[len(files)-1] + counts, err := processRawV8Coverage(latestFile) + if err != nil { + return nil, err + } + + // Delete all raw V8 files except the latest (c8 needs the final one for aggregate) + // Actually for aggregate we only need the last file, so we can delete earlier ones + for i := 0; i < len(files)-1; i++ { + os.Remove(files[i]) + } + + return counts, nil +} + +// ListV8CoverageFiles returns sorted list of V8 coverage files in the raw dir. +func (e *Executor) ListV8CoverageFiles() ([]string, error) { + entries, err := os.ReadDir(e.coverageRawDir) + if err != nil { + return nil, err + } + + var files []string + for _, entry := range entries { + if !entry.IsDir() && strings.HasPrefix(entry.Name(), "coverage-") && strings.HasSuffix(entry.Name(), ".json") { + files = append(files, filepath.Join(e.coverageRawDir, entry.Name())) + } + } + sort.Strings(files) + return files, nil +} + +// CoverageTestRecord tracks processed coverage data for a single test. +type CoverageTestRecord struct { + TestID string + TestName string + LineCounts map[string]map[string]int // Processed line counts from V8 snapshot (cumulative) +} + +// ProcessCoverage runs c8 report for aggregate and diffs pre-processed snapshots for per-test coverage. +// Records should include a baseline at index 0 followed by per-test records. +// Each record's LineCounts has already been extracted from raw V8 files during test execution. +func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { + if !e.coverageEnabled { + return nil + } + + log.Stderrln("\n➀ Processing coverage data...") + + // 1. Run c8 report for aggregate (uses remaining raw V8 files in raw dir) + aggregateDir := filepath.Join(e.coverageOutputDir, "aggregate") + if err := os.MkdirAll(aggregateDir, 0o750); err != nil { + return fmt.Errorf("failed to create aggregate dir: %w", err) + } + + absRawDir, _ := filepath.Abs(e.coverageRawDir) + if err := runC8Report(absRawDir, aggregateDir); err != nil { + return fmt.Errorf("aggregate c8 report failed: %w", err) + } + + // 2. Diff consecutive pre-processed snapshots for per-test coverage + // records[0] = baseline, records[1..N] = per-test + testRecords := make([]CoverageTestRecord, 0) + for i := 1; i < len(records); i++ { + prev := records[i-1].LineCounts + curr := records[i].LineCounts + + diff := DiffV8LineCounts(prev, curr) + + testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(records[i].TestID)) + if err := os.MkdirAll(testDir, 0o750); err != nil { + return err + } + + diffPath := filepath.Join(testDir, "coverage.json") + diffData, _ := json.MarshalIndent(diff, "", " ") + if err := os.WriteFile(diffPath, diffData, 0o644); err != nil { + return err + } + + testRecords = append(testRecords, records[i]) + } + + // 3. Clean up raw V8 files (aggregate already processed) + os.RemoveAll(e.coverageRawDir) + + // 4. Print summary + return e.printCoverageSummary(testRecords) +} + +// processRawV8Coverage runs the Node.js helper to extract line-level counts from a V8 coverage file. +func processRawV8Coverage(v8FilePath string) (map[string]map[string]int, error) { + absPath, _ := filepath.Abs(v8FilePath) + + // Find the helper script - it's embedded alongside the binary or in the source tree + scriptPath := findCoverageScript() + if scriptPath == "" { + return nil, fmt.Errorf("could not find process-v8-coverage.js helper script") + } + + sourceRoot, _ := os.Getwd() + cmd := exec.Command("node", scriptPath, absPath, sourceRoot) + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("V8 coverage processing failed: %w\nOutput: %s", err, string(output)) + } + + // Parse the output: { "/path/file.js": { "lines": { "10": 5, "11": 3 } } } + var raw map[string]struct { + Lines map[string]int `json:"lines"` + } + if err := json.Unmarshal(output, &raw); err != nil { + return nil, fmt.Errorf("failed to parse V8 coverage output: %w", err) + } + + result := make(map[string]map[string]int) + for filePath, data := range raw { + result[filePath] = data.Lines + } + return result, nil +} + +var cachedScriptPath string + +// findCoverageScript writes the embedded script to a temp file and returns its path. +func findCoverageScript() string { + if cachedScriptPath != "" { + if _, err := os.Stat(cachedScriptPath); err == nil { + return cachedScriptPath + } + } + + tmpFile, err := os.CreateTemp("", "tusk-v8-coverage-*.js") + if err != nil { + return "" + } + if _, err := tmpFile.WriteString(processV8CoverageScript); err != nil { + tmpFile.Close() + return "" + } + tmpFile.Close() + cachedScriptPath = tmpFile.Name() + return cachedScriptPath +} + +// DiffV8LineCounts computes which lines were newly executed between two V8 snapshots. +func DiffV8LineCounts(prev, curr map[string]map[string]int) map[string]CoverageFileDiff { + result := make(map[string]CoverageFileDiff) + + for filePath, currLines := range curr { + prevLines := prev[filePath] + + var coveredLines []int + for lineStr, currCount := range currLines { + prevCount := 0 + if prevLines != nil { + prevCount = prevLines[lineStr] + } + + if currCount > prevCount { + line := 0 + fmt.Sscanf(lineStr, "%d", &line) + if line > 0 { + coveredLines = append(coveredLines, line) + } + } + } + + if len(coveredLines) > 0 { + sort.Ints(coveredLines) + coveredLines = dedup(coveredLines) + result[filePath] = CoverageFileDiff{ + CoveredLines: coveredLines, + CoverableLines: len(currLines), + CoveredCount: len(coveredLines), + } + } + } + + return result +} + +// prepareSnapshotDir creates a temp dir with a single V8 coverage file for c8 to process. +func prepareSnapshotDir(v8FilePath string) (string, error) { + tempDir, err := os.MkdirTemp("", "tusk-coverage-snapshot-*") + if err != nil { + return "", err + } + + data, err := os.ReadFile(v8FilePath) + if err != nil { + os.RemoveAll(tempDir) + return "", err + } + + destPath := filepath.Join(tempDir, filepath.Base(v8FilePath)) + if err := os.WriteFile(destPath, data, 0o644); err != nil { + os.RemoveAll(tempDir) + return "", err + } + + return tempDir, nil +} + +// runC8Report runs npx c8 report on V8 coverage files in tempDir, outputting Istanbul JSON to outputDir. +func runC8Report(tempDir string, outputDir string) error { + absOutputDir, _ := filepath.Abs(outputDir) + absTempDir, _ := filepath.Abs(tempDir) + + cmd := exec.Command("npx", "c8", "report", + "--all", + "--temp-directory", absTempDir, + "--report-dir", absOutputDir, + "--reporter", "json", + ) + cmd.Dir = "." // Run from project root so --all finds source files + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("c8 report failed: %w\nOutput: %s", err, string(output)) + } + return nil +} + +// IstanbulCoverage represents the Istanbul JSON coverage format. +type IstanbulCoverage map[string]IstanbulFileCoverage + +type IstanbulFileCoverage struct { + Path string `json:"path"` + StatementMap map[string]IstanbulLocation `json:"statementMap"` + S map[string]int `json:"s"` + FnMap map[string]json.RawMessage `json:"fnMap,omitempty"` + F map[string]int `json:"f,omitempty"` + BranchMap map[string]json.RawMessage `json:"branchMap,omitempty"` + B map[string]json.RawMessage `json:"b,omitempty"` +} + +type IstanbulLocation struct { + Start IstanbulPosition `json:"start"` + End IstanbulPosition `json:"end"` +} + +type IstanbulPosition struct { + Line int `json:"line"` + Column int `json:"column"` +} + +// CoverageFileDiff represents per-test coverage for a single file. +type CoverageFileDiff struct { + CoveredLines []int `json:"covered_lines"` + CoverableLines int `json:"coverable_lines"` + CoveredCount int `json:"covered_count"` +} + +// diffIstanbulCoverage computes which statements were newly executed between two Istanbul JSON snapshots. +func diffIstanbulCoverage(prevPath, currPath string) (map[string]CoverageFileDiff, error) { + prev, err := loadIstanbulJSON(prevPath) + if err != nil { + return nil, fmt.Errorf("loading prev coverage: %w", err) + } + curr, err := loadIstanbulJSON(currPath) + if err != nil { + return nil, fmt.Errorf("loading curr coverage: %w", err) + } + + result := make(map[string]CoverageFileDiff) + + for filePath, currFile := range curr { + prevFile, hasPrev := prev[filePath] + + var coveredLines []int + coverableLines := len(currFile.S) + + for stmtID, currCount := range currFile.S { + prevCount := 0 + if hasPrev { + prevCount = prevFile.S[stmtID] + } + + // This statement was executed during this test (delta > 0) + if currCount > prevCount { + // Get the line number from statementMap + if loc, ok := currFile.StatementMap[stmtID]; ok { + coveredLines = append(coveredLines, loc.Start.Line) + } + } + } + + if len(coveredLines) > 0 { + sort.Ints(coveredLines) + // Deduplicate (multiple statements can be on same line) + coveredLines = dedup(coveredLines) + result[filePath] = CoverageFileDiff{ + CoveredLines: coveredLines, + CoverableLines: coverableLines, + CoveredCount: len(coveredLines), + } + } + } + + return result, nil +} + +func loadIstanbulJSON(path string) (IstanbulCoverage, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var cov IstanbulCoverage + if err := json.Unmarshal(data, &cov); err != nil { + return nil, err + } + return cov, nil +} + +func dedup(sorted []int) []int { + if len(sorted) == 0 { + return sorted + } + result := []int{sorted[0]} + for i := 1; i < len(sorted); i++ { + if sorted[i] != sorted[i-1] { + result = append(result, sorted[i]) + } + } + return result +} + +func sanitizeFileName(name string) string { + // Replace characters that are problematic in file paths + replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") + return replacer.Replace(name) +} + +// CoverageSummary is the final output written to summary.json. +type CoverageSummary struct { + Timestamp string `json:"timestamp"` + Aggregate CoverageAggregate `json:"aggregate"` + PerFile map[string]CoverageFileSummary `json:"per_file"` + PerTest []CoverageTestSummary `json:"per_test"` +} + +type CoverageAggregate struct { + TotalCoverableLines int `json:"total_coverable_lines"` + TotalCoveredLines int `json:"total_covered_lines"` + CoveragePct float64 `json:"coverage_pct"` + TotalFiles int `json:"total_files"` + CoveredFiles int `json:"covered_files"` +} + +type CoverageFileSummary struct { + CoveredLines int `json:"covered_lines"` + CoverableLines int `json:"coverable_lines"` + CoveragePct float64 `json:"coverage_pct"` +} + +type CoverageTestSummary struct { + TestID string `json:"test_id"` + TestName string `json:"test_name"` + CoveredLines int `json:"covered_lines"` + FilesTouched int `json:"files_touched"` +} + +func (e *Executor) printCoverageSummary(records []CoverageTestRecord) error { + // Load aggregate Istanbul JSON + aggregatePath := filepath.Join(e.coverageOutputDir, "aggregate", "coverage-final.json") + aggCov, err := loadIstanbulJSON(aggregatePath) + if err != nil { + return fmt.Errorf("failed to load aggregate coverage: %w", err) + } + + summary := CoverageSummary{ + Timestamp: time.Now().Format(time.RFC3339), + PerFile: make(map[string]CoverageFileSummary), + } + + totalCoverable := 0 + totalCovered := 0 + coveredFiles := 0 + + for filePath, fileCov := range aggCov { + coverable := len(fileCov.S) + covered := 0 + for _, count := range fileCov.S { + if count > 0 { + covered++ + } + } + + totalCoverable += coverable + totalCovered += covered + if covered > 0 { + coveredFiles++ + } + + pct := 0.0 + if coverable > 0 { + pct = float64(covered) / float64(coverable) * 100 + } + + summary.PerFile[filePath] = CoverageFileSummary{ + CoveredLines: covered, + CoverableLines: coverable, + CoveragePct: pct, + } + } + + aggPct := 0.0 + if totalCoverable > 0 { + aggPct = float64(totalCovered) / float64(totalCoverable) * 100 + } + + summary.Aggregate = CoverageAggregate{ + TotalCoverableLines: totalCoverable, + TotalCoveredLines: totalCovered, + CoveragePct: aggPct, + TotalFiles: len(aggCov), + CoveredFiles: coveredFiles, + } + + // Per-test summaries + for _, record := range records { + testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(record.TestID)) + diffPath := filepath.Join(testDir, "coverage.json") + + testSummary := CoverageTestSummary{ + TestID: record.TestID, + TestName: record.TestName, + } + + data, err := os.ReadFile(diffPath) + if err == nil { + var diff map[string]CoverageFileDiff + if json.Unmarshal(data, &diff) == nil { + totalLines := 0 + for _, fd := range diff { + totalLines += fd.CoveredCount + } + testSummary.CoveredLines = totalLines + testSummary.FilesTouched = len(diff) + } + } + + summary.PerTest = append(summary.PerTest, testSummary) + } + + // Write summary.json + summaryPath := filepath.Join(e.coverageOutputDir, "summary.json") + summaryData, _ := json.MarshalIndent(summary, "", " ") + if err := os.WriteFile(summaryPath, summaryData, 0o644); err != nil { + return err + } + + // Print to console + log.Stderrln(fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% (%d/%d coverable lines across %d files)", + aggPct, totalCovered, totalCoverable, len(aggCov))) + + // Print top files + type fileStat struct { + path string + pct float64 + cov int + tot int + } + var stats []fileStat + for fp, fs := range summary.PerFile { + if fs.CoverableLines > 0 { + stats = append(stats, fileStat{fp, fs.CoveragePct, fs.CoveredLines, fs.CoverableLines}) + } + } + sort.Slice(stats, func(i, j int) bool { return stats[i].pct > stats[j].pct }) + + log.Stderrln("\n Per-file:") + for _, s := range stats { + shortPath := s.path + if cwd, err := os.Getwd(); err == nil { + if rel, err := filepath.Rel(cwd, s.path); err == nil { + shortPath = rel + } + } + log.Stderrln(fmt.Sprintf(" %-40s %5.1f%% (%d/%d)", shortPath, s.pct, s.cov, s.tot)) + } + + log.Stderrln("\n Per-test:") + for _, ts := range summary.PerTest { + name := ts.TestName + if name == "" { + name = ts.TestID + } + log.Stderrln(fmt.Sprintf(" %-40s %d lines across %d files", name, ts.CoveredLines, ts.FilesTouched)) + } + + log.Stderrln(fmt.Sprintf("\n Full report: %s", summaryPath)) + + return nil +} diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 80f08cd..d45db04 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -92,6 +92,14 @@ type Executor struct { replayComposeOverride string replayEnvVars map[string]string replaySandboxConfigPath string + + // Coverage + coverageEnabled bool + coverageRawDir string // NODE_V8_COVERAGE output dir + coveragePort int // Coverage snapshot server port + coverageOutputDir string // Final processed coverage output dir + coveragePerTest map[string]map[string]CoverageFileDiff // testID -> per-file coverage diff + coveragePerTestMu sync.Mutex } func NewExecutor() *Executor { @@ -475,6 +483,38 @@ func (e *Executor) SetOnTestCompleted(callback func(TestResult, Test)) { e.OnTestCompleted = callback } +func (e *Executor) SetCoverageEnabled(enabled bool) { + e.coverageEnabled = enabled +} + +func (e *Executor) IsCoverageEnabled() bool { + return e.coverageEnabled +} + +func (e *Executor) GetCoverageOutputDir() string { + return e.coverageOutputDir +} + +// SetTestCoverageDetail stores per-test coverage diff for display in TUI/print. +func (e *Executor) SetTestCoverageDetail(testID string, detail map[string]CoverageFileDiff) { + e.coveragePerTestMu.Lock() + defer e.coveragePerTestMu.Unlock() + if e.coveragePerTest == nil { + e.coveragePerTest = make(map[string]map[string]CoverageFileDiff) + } + e.coveragePerTest[testID] = detail +} + +// GetTestCoverageDetail returns per-test coverage diff for a given test. +func (e *Executor) GetTestCoverageDetail(testID string) map[string]CoverageFileDiff { + e.coveragePerTestMu.Lock() + defer e.coveragePerTestMu.Unlock() + if e.coveragePerTest == nil { + return nil + } + return e.coveragePerTest[testID] +} + func (e *Executor) SetSuiteSpans(spans []*core.Span) { e.suiteSpans = spans if e.server != nil && len(spans) > 0 { diff --git a/internal/runner/scripts/process-v8-coverage.js b/internal/runner/scripts/process-v8-coverage.js new file mode 100644 index 0000000..d2c7953 --- /dev/null +++ b/internal/runner/scripts/process-v8-coverage.js @@ -0,0 +1,82 @@ +#!/usr/bin/env node +/** + * Processes raw V8 coverage files and outputs per-file statement counts. + * Usage: node process-v8-coverage.js [source-root] + * + * Outputs JSON: { "/path/to/file.js": { "statements": { "lineNumber": hitCount } } } + * Only includes user source files (excludes node_modules, node: builtins). + */ + +const fs = require('fs'); +const path = require('path'); + +const v8File = process.argv[2]; +const sourceRoot = process.argv[3] || process.cwd(); + +if (!v8File) { + console.error('Usage: node process-v8-coverage.js [source-root]'); + process.exit(1); +} + +const data = JSON.parse(fs.readFileSync(v8File, 'utf-8')); +const result = {}; + +for (const script of data.result) { + // Skip non-file URLs (node: builtins, eval, etc.) + if (!script.url.startsWith('file://')) continue; + + const filePath = script.url.replace('file://', ''); + + // Skip node_modules + if (filePath.includes('node_modules')) continue; + + // Skip files outside source root + if (!filePath.startsWith(sourceRoot)) continue; + + // Read source file to map byte offsets to line numbers + let source; + try { + source = fs.readFileSync(filePath, 'utf-8'); + } catch { + continue; + } + + // Build offset-to-line mapping + const lineStarts = [0]; + for (let i = 0; i < source.length; i++) { + if (source[i] === '\n') { + lineStarts.push(i + 1); + } + } + + function offsetToLine(offset) { + let lo = 0, hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (lineStarts[mid] <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; // 1-based + } + + const lineCounts = {}; + + for (const func of script.functions) { + for (const range of func.ranges) { + if (range.count === 0) continue; + + const startLine = offsetToLine(range.startOffset); + const endLine = offsetToLine(range.endOffset); + + for (let line = startLine; line <= endLine; line++) { + lineCounts[line] = (lineCounts[line] || 0) + range.count; + } + } + } + + if (Object.keys(lineCounts).length > 0) { + result[filePath] = { lines: lineCounts }; + } +} + +process.stdout.write(JSON.stringify(result)); diff --git a/internal/runner/service.go b/internal/runner/service.go index e8bd65d..ae633ec 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -146,6 +146,29 @@ func (e *Executor) StartService() error { } env = append(env, "TUSK_DRIFT_MODE=REPLAY") + + // Coverage: inject NODE_V8_COVERAGE and TUSK_COVERAGE_PORT + if e.coverageEnabled { + if e.coverageRawDir == "" { + timestamp := time.Now().Format("20060102T150405") + e.coverageRawDir = filepath.Join(".tusk", "coverage-raw-"+timestamp) + e.coverageOutputDir = filepath.Join(".tusk", "coverage-"+timestamp) + } + if err := os.MkdirAll(e.coverageRawDir, 0o750); err != nil { + return fmt.Errorf("failed to create coverage raw dir: %w", err) + } + if err := os.MkdirAll(e.coverageOutputDir, 0o750); err != nil { + return fmt.Errorf("failed to create coverage output dir: %w", err) + } + absCoverageRawDir, _ := filepath.Abs(e.coverageRawDir) + if e.coveragePort == 0 { + e.coveragePort = 19876 + } + env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", absCoverageRawDir)) + env = append(env, fmt.Sprintf("TUSK_COVERAGE_PORT=%d", e.coveragePort)) + log.Debug("Coverage enabled", "raw_dir", absCoverageRawDir, "port", e.coveragePort) + } + e.serviceCmd.Env = env // Always capture service logs during startup. diff --git a/internal/runner/types.go b/internal/runner/types.go index 93c2dd4..b67342b 100644 --- a/internal/runner/types.go +++ b/internal/runner/types.go @@ -44,6 +44,7 @@ type TestResult struct { Duration int `json:"duration"` // In milliseconds Deviations []Deviation `json:"deviations,omitempty"` Error string `json:"error,omitempty"` + } type Trace struct { diff --git a/internal/tui/test_executor.go b/internal/tui/test_executor.go index 1619ff2..06975b8 100644 --- a/internal/tui/test_executor.go +++ b/internal/tui/test_executor.go @@ -6,6 +6,7 @@ import ( "io" "log/slog" "os" + "path/filepath" "slices" "strings" "time" @@ -620,6 +621,30 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if m.executor.OnTestCompleted != nil { m.executor.OnTestCompleted(msg.result, test) } + + // Show per-file coverage breakdown in test log panel + if m.executor.IsCoverageEnabled() { + if detail := m.executor.GetTestCoverageDetail(test.TraceID); len(detail) > 0 { + totalLines := 0 + for _, fd := range detail { + totalLines += fd.CoveredCount + } + m.addTestLog(test.TraceID, fmt.Sprintf(" πŸ“Š Coverage: %d lines across %d files", totalLines, len(detail))) + for filePath, fd := range detail { + // Shorten the file path relative to cwd + shortPath := filePath + if cwd, err := os.Getwd(); err == nil { + if rel, err := filepath.Rel(cwd, filePath); err == nil { + shortPath = rel + } + } + m.addTestLog(test.TraceID, fmt.Sprintf(" %-40s %d lines", shortPath, fd.CoveredCount)) + } + } else { + m.addTestLog(test.TraceID, " πŸ“Š Coverage: 0 new lines") + } + } + if m.opts != nil && m.opts.OnTestCompleted != nil { res := msg.result go m.opts.OnTestCompleted(res, test, m.executor) From a812bc702806655f14d38da77040ffd3935d2514 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Tue, 31 Mar 2026 23:11:22 -0700 Subject: [PATCH 02/36] refactor: revert to v8.takeCoverage() approach, add TUI coverage display V8 Inspector precise coverage doesn't work for per-test tracking because takePreciseCoverage only returns scripts loaded AFTER startPreciseCoverage is called. Since user code loads at startup before SDK init, the Inspector misses all user scripts. NODE_V8_COVERAGE best-effort coverage is binary (0/1 counts), so per-test diffing gives marginal coverage (newly covered lines only). Changes: - Reverted to v8.takeCoverage() + Node.js helper for per-test processing - Added per-file coverage breakdown in TUI test log panel - Added coverage sub-lines in --print mode - Stored per-test coverage diffs on executor for TUI access --- cmd/run.go | 25 +++++---- internal/runner/coverage.go | 93 ++++++++++++++++++++-------------- scripts/process-v8-coverage.js | 82 ++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 48 deletions(-) create mode 100644 scripts/process-v8-coverage.js diff --git a/cmd/run.go b/cmd/run.go index b2b46e1..8183205 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -410,7 +410,8 @@ func runTests(cmd *cobra.Command, args []string) error { existingCallback(res, test) } - // Take coverage snapshot and process immediately (extract line counts, clean up raw V8 file) + // Take coverage snapshot - SDK returns cumulative precise counts + // We diff with previous snapshot to get true per-test coverage lineCounts, err := executor.TakeCoverageSnapshotAndProcess() if err != nil { log.Warn("Failed to take coverage snapshot", "testID", test.TraceID, "error", err) @@ -418,7 +419,7 @@ func runTests(cmd *cobra.Command, args []string) error { } coverageMu.Lock() - // Compute per-test diff (delta from previous snapshot) + // Diff with previous snapshot (precise counts, not binary) var prevCounts map[string]map[string]int if len(coverageRecords) > 0 { prevCounts = coverageRecords[len(coverageRecords)-1].LineCounts @@ -430,7 +431,7 @@ func runTests(cmd *cobra.Command, args []string) error { }) coverageMu.Unlock() - // Diff with previous snapshot to get this test's coverage + // Diff gives us lines where count increased = lines this test executed diff := runner.DiffV8LineCounts(prevCounts, lineCounts) executor.SetTestCoverageDetail(test.TraceID, diff) @@ -880,9 +881,10 @@ func runTests(cmd *cobra.Command, args []string) error { log.Stderrln(fmt.Sprintf("➀ Running %d tests (concurrency: %d)...\n", len(tests), executor.GetConcurrency())) } - // Coverage: take baseline snapshot before any tests run + // Coverage: take baseline snapshot (captures startup coverage) + // This becomes the reference for diffing the first test's coverage if coverageEnabled { - // Small delay to let the service fully initialize and coverage server start + // Small delay to let the coverage server start time.Sleep(500 * time.Millisecond) baselineCounts, err := executor.TakeCoverageSnapshotAndProcess() if err != nil { @@ -895,7 +897,7 @@ func runTests(cmd *cobra.Command, args []string) error { LineCounts: baselineCounts, }) coverageMu.Unlock() - log.Debug("Coverage baseline snapshot taken") + log.Debug("Coverage baseline snapshot taken (precise mode)") } } @@ -929,15 +931,16 @@ func runTests(cmd *cobra.Command, args []string) error { _ = os.Stdout.Sync() time.Sleep(1 * time.Millisecond) - // Coverage: process coverage data after all tests complete (before service stops) + // Coverage: process aggregate and write per-test files + // records[0] = baseline, records[1..N] = per-test (with cumulative counts) if coverageEnabled && len(coverageRecords) > 1 { - // records[0] = baseline, records[1..N] = per-test (LineCounts already processed inline) coverageMu.Lock() - records := make([]runner.CoverageTestRecord, len(coverageRecords)) - copy(records, coverageRecords) + // Skip baseline, pass only test records (diffs already computed inline) + testRecords := make([]runner.CoverageTestRecord, 0, len(coverageRecords)-1) + testRecords = append(testRecords, coverageRecords[1:]...) coverageMu.Unlock() - if err := executor.ProcessCoverage(records); err != nil { + if err := executor.ProcessCoverage(testRecords); err != nil { log.Warn("Failed to process coverage", "error", err) } } diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index c753266..b9fdef5 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -21,39 +21,35 @@ var processV8CoverageScript string const coverageSnapshotTimeout = 5 * time.Second -// TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. -func (e *Executor) TakeCoverageSnapshot() error { +// TakeCoverageSnapshotAndProcess calls the SDK's coverage snapshot endpoint, +// processes the latest V8 file into compact line counts, and cleans up old files. +// Returns per-file line counts: filePath -> lineNumber -> hitCount +// Note: V8 best-effort coverage uses binary counts (0/1), so per-test diffing +// gives marginal coverage (newly covered lines), not total per-test coverage. +func (e *Executor) TakeCoverageSnapshotAndProcess() (map[string]map[string]int, error) { if !e.coverageEnabled || e.coveragePort == 0 { - return nil + return nil, nil } + // Call SDK to trigger v8.takeCoverage() url := fmt.Sprintf("http://127.0.0.1:%d/snapshot", e.coveragePort) - client := &http.Client{Timeout: coverageSnapshotTimeout} + httpClient := &http.Client{Timeout: coverageSnapshotTimeout} - resp, err := client.Get(url) + resp, err := httpClient.Get(url) if err != nil { - return fmt.Errorf("failed to take coverage snapshot: %w", err) + return nil, fmt.Errorf("failed to take coverage snapshot: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("coverage snapshot returned status %d: %s", resp.StatusCode, string(body)) + return nil, fmt.Errorf("coverage snapshot returned status %d: %s", resp.StatusCode, string(body)) } // Small delay to let V8 finish writing the file time.Sleep(100 * time.Millisecond) - return nil -} - -// TakeCoverageSnapshotAndProcess takes a snapshot, processes the latest V8 file -// into compact line counts, and deletes the raw V8 file to save disk space. -// Returns the processed line counts. -func (e *Executor) TakeCoverageSnapshotAndProcess() (map[string]map[string]int, error) { - if err := e.TakeCoverageSnapshot(); err != nil { - return nil, err - } + // Process the latest V8 file files, err := e.ListV8CoverageFiles() if err != nil { return nil, err @@ -62,15 +58,13 @@ func (e *Executor) TakeCoverageSnapshotAndProcess() (map[string]map[string]int, return make(map[string]map[string]int), nil } - // Process the latest file (cumulative snapshot) latestFile := files[len(files)-1] counts, err := processRawV8Coverage(latestFile) if err != nil { return nil, err } - // Delete all raw V8 files except the latest (c8 needs the final one for aggregate) - // Actually for aggregate we only need the last file, so we can delete earlier ones + // Delete old V8 files to save disk space (keep latest for c8 aggregate) for i := 0; i < len(files)-1; i++ { os.Remove(files[i]) } @@ -102,9 +96,9 @@ type CoverageTestRecord struct { LineCounts map[string]map[string]int // Processed line counts from V8 snapshot (cumulative) } -// ProcessCoverage runs c8 report for aggregate and diffs pre-processed snapshots for per-test coverage. -// Records should include a baseline at index 0 followed by per-test records. -// Each record's LineCounts has already been extracted from raw V8 files during test execution. +// ProcessCoverage runs c8 report for aggregate and writes per-test coverage files. +// With V8 Inspector precise coverage, each record already contains clean per-test +// coverage (counters reset between tests), so no diffing is needed. func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { if !e.coverageEnabled { return nil @@ -112,7 +106,7 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { log.Stderrln("\n➀ Processing coverage data...") - // 1. Run c8 report for aggregate (uses remaining raw V8 files in raw dir) + // 1. Run c8 report for aggregate (uses NODE_V8_COVERAGE files for --all support) aggregateDir := filepath.Join(e.coverageOutputDir, "aggregate") if err := os.MkdirAll(aggregateDir, 0o750); err != nil { return fmt.Errorf("failed to create aggregate dir: %w", err) @@ -123,34 +117,59 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { return fmt.Errorf("aggregate c8 report failed: %w", err) } - // 2. Diff consecutive pre-processed snapshots for per-test coverage - // records[0] = baseline, records[1..N] = per-test - testRecords := make([]CoverageTestRecord, 0) - for i := 1; i < len(records); i++ { - prev := records[i-1].LineCounts - curr := records[i].LineCounts - - diff := DiffV8LineCounts(prev, curr) + // 2. Write per-test coverage files (diffs already computed during test execution) + for _, record := range records { + detail := e.GetTestCoverageDetail(record.TestID) + if detail == nil { + continue + } - testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(records[i].TestID)) + testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(record.TestID)) if err := os.MkdirAll(testDir, 0o750); err != nil { return err } diffPath := filepath.Join(testDir, "coverage.json") - diffData, _ := json.MarshalIndent(diff, "", " ") + diffData, _ := json.MarshalIndent(detail, "", " ") if err := os.WriteFile(diffPath, diffData, 0o644); err != nil { return err } - - testRecords = append(testRecords, records[i]) } // 3. Clean up raw V8 files (aggregate already processed) os.RemoveAll(e.coverageRawDir) // 4. Print summary - return e.printCoverageSummary(testRecords) + return e.printCoverageSummary(records) +} + +// LinecountsToCoverageDetail converts raw line counts to CoverageFileDiff format. +func LinecountsToCoverageDetail(lineCounts map[string]map[string]int) map[string]CoverageFileDiff { + result := make(map[string]CoverageFileDiff) + + for filePath, lines := range lineCounts { + var coveredLines []int + for lineStr, count := range lines { + if count > 0 { + line := 0 + fmt.Sscanf(lineStr, "%d", &line) + if line > 0 { + coveredLines = append(coveredLines, line) + } + } + } + if len(coveredLines) > 0 { + sort.Ints(coveredLines) + coveredLines = dedup(coveredLines) + result[filePath] = CoverageFileDiff{ + CoveredLines: coveredLines, + CoverableLines: len(lines), + CoveredCount: len(coveredLines), + } + } + } + + return result } // processRawV8Coverage runs the Node.js helper to extract line-level counts from a V8 coverage file. diff --git a/scripts/process-v8-coverage.js b/scripts/process-v8-coverage.js new file mode 100644 index 0000000..d2c7953 --- /dev/null +++ b/scripts/process-v8-coverage.js @@ -0,0 +1,82 @@ +#!/usr/bin/env node +/** + * Processes raw V8 coverage files and outputs per-file statement counts. + * Usage: node process-v8-coverage.js [source-root] + * + * Outputs JSON: { "/path/to/file.js": { "statements": { "lineNumber": hitCount } } } + * Only includes user source files (excludes node_modules, node: builtins). + */ + +const fs = require('fs'); +const path = require('path'); + +const v8File = process.argv[2]; +const sourceRoot = process.argv[3] || process.cwd(); + +if (!v8File) { + console.error('Usage: node process-v8-coverage.js [source-root]'); + process.exit(1); +} + +const data = JSON.parse(fs.readFileSync(v8File, 'utf-8')); +const result = {}; + +for (const script of data.result) { + // Skip non-file URLs (node: builtins, eval, etc.) + if (!script.url.startsWith('file://')) continue; + + const filePath = script.url.replace('file://', ''); + + // Skip node_modules + if (filePath.includes('node_modules')) continue; + + // Skip files outside source root + if (!filePath.startsWith(sourceRoot)) continue; + + // Read source file to map byte offsets to line numbers + let source; + try { + source = fs.readFileSync(filePath, 'utf-8'); + } catch { + continue; + } + + // Build offset-to-line mapping + const lineStarts = [0]; + for (let i = 0; i < source.length; i++) { + if (source[i] === '\n') { + lineStarts.push(i + 1); + } + } + + function offsetToLine(offset) { + let lo = 0, hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (lineStarts[mid] <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; // 1-based + } + + const lineCounts = {}; + + for (const func of script.functions) { + for (const range of func.ranges) { + if (range.count === 0) continue; + + const startLine = offsetToLine(range.startOffset); + const endLine = offsetToLine(range.endOffset); + + for (let line = startLine; line <= endLine; line++) { + lineCounts[line] = (lineCounts[line] || 0) + range.count; + } + } + } + + if (Object.keys(lineCounts).length > 0) { + result[filePath] = { lines: lineCounts }; + } +} + +process.stdout.write(JSON.stringify(result)); From 81ddfa5fa33a53fd8602b4c58e7bbf80fac772cb Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 00:41:12 -0700 Subject: [PATCH 03/36] feat: switch coverage from NYC to V8 native approach Major simplification of coverage implementation: - Remove NYC command wrapping (resolveNpmScript, etc.) - Just set NODE_V8_COVERAGE + TUSK_COVERAGE_PORT env vars - v8.takeCoverage() auto-resets counters -> each snapshot is clean per-test data - No diffing needed (was needed for NYC cumulative approach) - Aggregate computed by merging all per-test snapshots - Remove process-v8-coverage.js helper script (SDK handles V8 processing) - Remove NYC-specific code (command resolution, Istanbul JSON parsing) - Works with any start command (npm, yarn, docker, shell scripts) --- cmd/run.go | 53 +- internal/runner/coverage.go | 495 ++++-------------- internal/runner/executor.go | 3 +- .../runner/scripts/process-v8-coverage.js | 82 --- internal/runner/service.go | 39 +- scripts/process-v8-coverage.js | 82 --- 6 files changed, 150 insertions(+), 604 deletions(-) delete mode 100644 internal/runner/scripts/process-v8-coverage.js delete mode 100644 scripts/process-v8-coverage.js diff --git a/cmd/run.go b/cmd/run.go index 8183205..fd0deec 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -405,25 +405,19 @@ func runTests(cmd *cobra.Command, args []string) error { if coverageEnabled { existingCallback := executor.OnTestCompleted executor.SetOnTestCompleted(func(res runner.TestResult, test runner.Test) { - // Call the original callback first if existingCallback != nil { existingCallback(res, test) } - // Take coverage snapshot - SDK returns cumulative precise counts - // We diff with previous snapshot to get true per-test coverage - lineCounts, err := executor.TakeCoverageSnapshotAndProcess() + // Take coverage snapshot. V8's takeCoverage() auto-resets counters, + // so the response contains ONLY coverage from this test. + lineCounts, err := executor.TakeCoverageSnapshot() if err != nil { log.Warn("Failed to take coverage snapshot", "testID", test.TraceID, "error", err) return } coverageMu.Lock() - // Diff with previous snapshot (precise counts, not binary) - var prevCounts map[string]map[string]int - if len(coverageRecords) > 0 { - prevCounts = coverageRecords[len(coverageRecords)-1].LineCounts - } coverageRecords = append(coverageRecords, runner.CoverageTestRecord{ TestID: test.TraceID, TestName: fmt.Sprintf("%s %s", test.Method, test.Path), @@ -431,18 +425,18 @@ func runTests(cmd *cobra.Command, args []string) error { }) coverageMu.Unlock() - // Diff gives us lines where count increased = lines this test executed - diff := runner.DiffV8LineCounts(prevCounts, lineCounts) - executor.SetTestCoverageDetail(test.TraceID, diff) + // Store detail for TUI display + detail := runner.LinecountsToCoverageDetail(lineCounts) + executor.SetTestCoverageDetail(test.TraceID, detail) // Print sub-line in --print mode if !interactive { totalLines := 0 - for _, fd := range diff { + for _, fd := range detail { totalLines += fd.CoveredCount } if totalLines > 0 { - log.UserProgress(fmt.Sprintf(" ↳ coverage: %d lines across %d files", totalLines, len(diff))) + log.UserProgress(fmt.Sprintf(" ↳ coverage: %d lines across %d files", totalLines, len(detail))) } } }) @@ -881,23 +875,14 @@ func runTests(cmd *cobra.Command, args []string) error { log.Stderrln(fmt.Sprintf("➀ Running %d tests (concurrency: %d)...\n", len(tests), executor.GetConcurrency())) } - // Coverage: take baseline snapshot (captures startup coverage) - // This becomes the reference for diffing the first test's coverage + // Coverage: take baseline snapshot to discard startup coverage. + // v8.takeCoverage() resets counters, so the first real test gets clean data. if coverageEnabled { - // Small delay to let the coverage server start - time.Sleep(500 * time.Millisecond) - baselineCounts, err := executor.TakeCoverageSnapshotAndProcess() - if err != nil { + time.Sleep(500 * time.Millisecond) // let coverage server start + if _, err := executor.TakeCoverageSnapshot(); err != nil { log.Warn("Failed to take baseline coverage snapshot", "error", err) } else { - coverageMu.Lock() - coverageRecords = append(coverageRecords, runner.CoverageTestRecord{ - TestID: "_baseline", - TestName: "baseline", - LineCounts: baselineCounts, - }) - coverageMu.Unlock() - log.Debug("Coverage baseline snapshot taken (precise mode)") + log.Debug("Coverage baseline taken (startup coverage discarded, counters reset)") } } @@ -931,16 +916,14 @@ func runTests(cmd *cobra.Command, args []string) error { _ = os.Stdout.Sync() time.Sleep(1 * time.Millisecond) - // Coverage: process aggregate and write per-test files - // records[0] = baseline, records[1..N] = per-test (with cumulative counts) - if coverageEnabled && len(coverageRecords) > 1 { + // Coverage: write per-test files and print summary + if coverageEnabled && len(coverageRecords) > 0 { coverageMu.Lock() - // Skip baseline, pass only test records (diffs already computed inline) - testRecords := make([]runner.CoverageTestRecord, 0, len(coverageRecords)-1) - testRecords = append(testRecords, coverageRecords[1:]...) + records := make([]runner.CoverageTestRecord, len(coverageRecords)) + copy(records, coverageRecords) coverageMu.Unlock() - if err := executor.ProcessCoverage(testRecords); err != nil { + if err := executor.ProcessCoverage(records); err != nil { log.Warn("Failed to process coverage", "error", err) } } diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index b9fdef5..1ad634a 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -1,13 +1,11 @@ package runner import ( - _ "embed" "encoding/json" "fmt" "io" "net/http" "os" - "os/exec" "path/filepath" "sort" "strings" @@ -16,411 +14,147 @@ import ( "github.com/Use-Tusk/tusk-cli/internal/log" ) -//go:embed scripts/process-v8-coverage.js -var processV8CoverageScript string - const coverageSnapshotTimeout = 5 * time.Second -// TakeCoverageSnapshotAndProcess calls the SDK's coverage snapshot endpoint, -// processes the latest V8 file into compact line counts, and cleans up old files. -// Returns per-file line counts: filePath -> lineNumber -> hitCount -// Note: V8 best-effort coverage uses binary counts (0/1), so per-test diffing -// gives marginal coverage (newly covered lines), not total per-test coverage. -func (e *Executor) TakeCoverageSnapshotAndProcess() (map[string]map[string]int, error) { +// TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. +// The SDK calls v8.takeCoverage() (Node) or coverage.py stop/start (Python), +// processes the coverage data, and returns per-file line counts. +// V8's takeCoverage() auto-resets counters, so each call returns ONLY the +// coverage since the last call - true per-test coverage with no diffing needed. +func (e *Executor) TakeCoverageSnapshot() (map[string]map[string]int, error) { if !e.coverageEnabled || e.coveragePort == 0 { return nil, nil } - // Call SDK to trigger v8.takeCoverage() url := fmt.Sprintf("http://127.0.0.1:%d/snapshot", e.coveragePort) httpClient := &http.Client{Timeout: coverageSnapshotTimeout} resp, err := httpClient.Get(url) if err != nil { - return nil, fmt.Errorf("failed to take coverage snapshot: %w", err) + return nil, fmt.Errorf("coverage snapshot failed: %w", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("coverage snapshot returned status %d: %s", resp.StatusCode, string(body)) - } - - // Small delay to let V8 finish writing the file - time.Sleep(100 * time.Millisecond) - - // Process the latest V8 file - files, err := e.ListV8CoverageFiles() + body, err := io.ReadAll(resp.Body) if err != nil { - return nil, err - } - if len(files) == 0 { - return make(map[string]map[string]int), nil + return nil, fmt.Errorf("failed to read coverage response: %w", err) } - latestFile := files[len(files)-1] - counts, err := processRawV8Coverage(latestFile) - if err != nil { - return nil, err + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("coverage snapshot status %d: %s", resp.StatusCode, string(body)) } - // Delete old V8 files to save disk space (keep latest for c8 aggregate) - for i := 0; i < len(files)-1; i++ { - os.Remove(files[i]) + var result struct { + OK bool `json:"ok"` + Coverage map[string]map[string]int `json:"coverage"` } - - return counts, nil -} - -// ListV8CoverageFiles returns sorted list of V8 coverage files in the raw dir. -func (e *Executor) ListV8CoverageFiles() ([]string, error) { - entries, err := os.ReadDir(e.coverageRawDir) - if err != nil { - return nil, err + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to parse coverage response: %w", err) } - var files []string - for _, entry := range entries { - if !entry.IsDir() && strings.HasPrefix(entry.Name(), "coverage-") && strings.HasSuffix(entry.Name(), ".json") { - files = append(files, filepath.Join(e.coverageRawDir, entry.Name())) - } + if !result.OK { + return nil, fmt.Errorf("coverage snapshot returned ok=false") } - sort.Strings(files) - return files, nil + + return result.Coverage, nil } -// CoverageTestRecord tracks processed coverage data for a single test. +// CoverageTestRecord holds per-test coverage data. type CoverageTestRecord struct { - TestID string - TestName string - LineCounts map[string]map[string]int // Processed line counts from V8 snapshot (cumulative) + TestID string + TestName string + LineCounts map[string]map[string]int // filePath -> lineNumber -> hitCount (per-test, not cumulative) } -// ProcessCoverage runs c8 report for aggregate and writes per-test coverage files. -// With V8 Inspector precise coverage, each record already contains clean per-test -// coverage (counters reset between tests), so no diffing is needed. -func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { - if !e.coverageEnabled { - return nil - } - - log.Stderrln("\n➀ Processing coverage data...") - - // 1. Run c8 report for aggregate (uses NODE_V8_COVERAGE files for --all support) - aggregateDir := filepath.Join(e.coverageOutputDir, "aggregate") - if err := os.MkdirAll(aggregateDir, 0o750); err != nil { - return fmt.Errorf("failed to create aggregate dir: %w", err) - } - - absRawDir, _ := filepath.Abs(e.coverageRawDir) - if err := runC8Report(absRawDir, aggregateDir); err != nil { - return fmt.Errorf("aggregate c8 report failed: %w", err) - } - - // 2. Write per-test coverage files (diffs already computed during test execution) - for _, record := range records { - detail := e.GetTestCoverageDetail(record.TestID) - if detail == nil { - continue - } - - testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(record.TestID)) - if err := os.MkdirAll(testDir, 0o750); err != nil { - return err - } - - diffPath := filepath.Join(testDir, "coverage.json") - diffData, _ := json.MarshalIndent(detail, "", " ") - if err := os.WriteFile(diffPath, diffData, 0o644); err != nil { - return err - } - } - - // 3. Clean up raw V8 files (aggregate already processed) - os.RemoveAll(e.coverageRawDir) - - // 4. Print summary - return e.printCoverageSummary(records) +// CoverageFileDiff represents per-test coverage for a single file. +type CoverageFileDiff struct { + CoveredLines []int `json:"covered_lines"` + CoverableLines int `json:"coverable_lines"` + CoveredCount int `json:"covered_count"` } // LinecountsToCoverageDetail converts raw line counts to CoverageFileDiff format. func LinecountsToCoverageDetail(lineCounts map[string]map[string]int) map[string]CoverageFileDiff { result := make(map[string]CoverageFileDiff) - for filePath, lines := range lineCounts { - var coveredLines []int + var covered []int for lineStr, count := range lines { if count > 0 { line := 0 fmt.Sscanf(lineStr, "%d", &line) if line > 0 { - coveredLines = append(coveredLines, line) + covered = append(covered, line) } } } - if len(coveredLines) > 0 { - sort.Ints(coveredLines) - coveredLines = dedup(coveredLines) + if len(covered) > 0 { + sort.Ints(covered) + covered = dedup(covered) result[filePath] = CoverageFileDiff{ - CoveredLines: coveredLines, + CoveredLines: covered, CoverableLines: len(lines), - CoveredCount: len(coveredLines), + CoveredCount: len(covered), } } } - return result } -// processRawV8Coverage runs the Node.js helper to extract line-level counts from a V8 coverage file. -func processRawV8Coverage(v8FilePath string) (map[string]map[string]int, error) { - absPath, _ := filepath.Abs(v8FilePath) - - // Find the helper script - it's embedded alongside the binary or in the source tree - scriptPath := findCoverageScript() - if scriptPath == "" { - return nil, fmt.Errorf("could not find process-v8-coverage.js helper script") - } - - sourceRoot, _ := os.Getwd() - cmd := exec.Command("node", scriptPath, absPath, sourceRoot) - output, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("V8 coverage processing failed: %w\nOutput: %s", err, string(output)) - } - - // Parse the output: { "/path/file.js": { "lines": { "10": 5, "11": 3 } } } - var raw map[string]struct { - Lines map[string]int `json:"lines"` - } - if err := json.Unmarshal(output, &raw); err != nil { - return nil, fmt.Errorf("failed to parse V8 coverage output: %w", err) - } - - result := make(map[string]map[string]int) - for filePath, data := range raw { - result[filePath] = data.Lines +// ProcessCoverage writes per-test coverage files and prints the summary. +func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { + if !e.coverageEnabled || len(records) == 0 { + return nil } - return result, nil -} -var cachedScriptPath string + log.Stderrln("\n➀ Processing coverage data...") -// findCoverageScript writes the embedded script to a temp file and returns its path. -func findCoverageScript() string { - if cachedScriptPath != "" { - if _, err := os.Stat(cachedScriptPath); err == nil { - return cachedScriptPath + // Write per-test coverage files + for _, record := range records { + detail := e.GetTestCoverageDetail(record.TestID) + if detail == nil { + continue } - } - - tmpFile, err := os.CreateTemp("", "tusk-v8-coverage-*.js") - if err != nil { - return "" - } - if _, err := tmpFile.WriteString(processV8CoverageScript); err != nil { - tmpFile.Close() - return "" - } - tmpFile.Close() - cachedScriptPath = tmpFile.Name() - return cachedScriptPath -} - -// DiffV8LineCounts computes which lines were newly executed between two V8 snapshots. -func DiffV8LineCounts(prev, curr map[string]map[string]int) map[string]CoverageFileDiff { - result := make(map[string]CoverageFileDiff) - - for filePath, currLines := range curr { - prevLines := prev[filePath] - - var coveredLines []int - for lineStr, currCount := range currLines { - prevCount := 0 - if prevLines != nil { - prevCount = prevLines[lineStr] - } - - if currCount > prevCount { - line := 0 - fmt.Sscanf(lineStr, "%d", &line) - if line > 0 { - coveredLines = append(coveredLines, line) - } - } + testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(record.TestID)) + if err := os.MkdirAll(testDir, 0o750); err != nil { + return err } - - if len(coveredLines) > 0 { - sort.Ints(coveredLines) - coveredLines = dedup(coveredLines) - result[filePath] = CoverageFileDiff{ - CoveredLines: coveredLines, - CoverableLines: len(currLines), - CoveredCount: len(coveredLines), - } + data, _ := json.MarshalIndent(detail, "", " ") + if err := os.WriteFile(filepath.Join(testDir, "coverage.json"), data, 0o644); err != nil { + return err } } - return result -} + // Compute aggregate by merging all per-test coverage (union of covered lines) + aggregate := mergeLineCounts(records) -// prepareSnapshotDir creates a temp dir with a single V8 coverage file for c8 to process. -func prepareSnapshotDir(v8FilePath string) (string, error) { - tempDir, err := os.MkdirTemp("", "tusk-coverage-snapshot-*") - if err != nil { - return "", err - } - - data, err := os.ReadFile(v8FilePath) - if err != nil { - os.RemoveAll(tempDir) - return "", err - } - - destPath := filepath.Join(tempDir, filepath.Base(v8FilePath)) - if err := os.WriteFile(destPath, data, 0o644); err != nil { - os.RemoveAll(tempDir) - return "", err - } - - return tempDir, nil -} - -// runC8Report runs npx c8 report on V8 coverage files in tempDir, outputting Istanbul JSON to outputDir. -func runC8Report(tempDir string, outputDir string) error { - absOutputDir, _ := filepath.Abs(outputDir) - absTempDir, _ := filepath.Abs(tempDir) - - cmd := exec.Command("npx", "c8", "report", - "--all", - "--temp-directory", absTempDir, - "--report-dir", absOutputDir, - "--reporter", "json", - ) - cmd.Dir = "." // Run from project root so --all finds source files - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("c8 report failed: %w\nOutput: %s", err, string(output)) - } - return nil -} - -// IstanbulCoverage represents the Istanbul JSON coverage format. -type IstanbulCoverage map[string]IstanbulFileCoverage - -type IstanbulFileCoverage struct { - Path string `json:"path"` - StatementMap map[string]IstanbulLocation `json:"statementMap"` - S map[string]int `json:"s"` - FnMap map[string]json.RawMessage `json:"fnMap,omitempty"` - F map[string]int `json:"f,omitempty"` - BranchMap map[string]json.RawMessage `json:"branchMap,omitempty"` - B map[string]json.RawMessage `json:"b,omitempty"` -} - -type IstanbulLocation struct { - Start IstanbulPosition `json:"start"` - End IstanbulPosition `json:"end"` -} - -type IstanbulPosition struct { - Line int `json:"line"` - Column int `json:"column"` -} - -// CoverageFileDiff represents per-test coverage for a single file. -type CoverageFileDiff struct { - CoveredLines []int `json:"covered_lines"` - CoverableLines int `json:"coverable_lines"` - CoveredCount int `json:"covered_count"` + // Print and write summary + return e.printCoverageSummary(records, aggregate) } -// diffIstanbulCoverage computes which statements were newly executed between two Istanbul JSON snapshots. -func diffIstanbulCoverage(prevPath, currPath string) (map[string]CoverageFileDiff, error) { - prev, err := loadIstanbulJSON(prevPath) - if err != nil { - return nil, fmt.Errorf("loading prev coverage: %w", err) - } - curr, err := loadIstanbulJSON(currPath) - if err != nil { - return nil, fmt.Errorf("loading curr coverage: %w", err) - } - - result := make(map[string]CoverageFileDiff) - - for filePath, currFile := range curr { - prevFile, hasPrev := prev[filePath] - - var coveredLines []int - coverableLines := len(currFile.S) - - for stmtID, currCount := range currFile.S { - prevCount := 0 - if hasPrev { - prevCount = prevFile.S[stmtID] +// mergeLineCounts unions all per-test line counts into an aggregate. +// A line's count in the aggregate = sum of counts across all tests. +func mergeLineCounts(records []CoverageTestRecord) map[string]map[string]int { + merged := make(map[string]map[string]int) + for _, record := range records { + for filePath, lines := range record.LineCounts { + if merged[filePath] == nil { + merged[filePath] = make(map[string]int) } - - // This statement was executed during this test (delta > 0) - if currCount > prevCount { - // Get the line number from statementMap - if loc, ok := currFile.StatementMap[stmtID]; ok { - coveredLines = append(coveredLines, loc.Start.Line) - } - } - } - - if len(coveredLines) > 0 { - sort.Ints(coveredLines) - // Deduplicate (multiple statements can be on same line) - coveredLines = dedup(coveredLines) - result[filePath] = CoverageFileDiff{ - CoveredLines: coveredLines, - CoverableLines: coverableLines, - CoveredCount: len(coveredLines), + for line, count := range lines { + merged[filePath][line] += count } } } - - return result, nil + return merged } -func loadIstanbulJSON(path string) (IstanbulCoverage, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var cov IstanbulCoverage - if err := json.Unmarshal(data, &cov); err != nil { - return nil, err - } - return cov, nil -} +// --- Summary output --- -func dedup(sorted []int) []int { - if len(sorted) == 0 { - return sorted - } - result := []int{sorted[0]} - for i := 1; i < len(sorted); i++ { - if sorted[i] != sorted[i-1] { - result = append(result, sorted[i]) - } - } - return result -} - -func sanitizeFileName(name string) string { - // Replace characters that are problematic in file paths - replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") - return replacer.Replace(name) -} - -// CoverageSummary is the final output written to summary.json. type CoverageSummary struct { - Timestamp string `json:"timestamp"` - Aggregate CoverageAggregate `json:"aggregate"` - PerFile map[string]CoverageFileSummary `json:"per_file"` - PerTest []CoverageTestSummary `json:"per_test"` + Timestamp string `json:"timestamp"` + Aggregate CoverageAggregate `json:"aggregate"` + PerFile map[string]CoverageFileSummary `json:"per_file"` + PerTest []CoverageTestSummary `json:"per_test"` } type CoverageAggregate struct { @@ -444,14 +178,7 @@ type CoverageTestSummary struct { FilesTouched int `json:"files_touched"` } -func (e *Executor) printCoverageSummary(records []CoverageTestRecord) error { - // Load aggregate Istanbul JSON - aggregatePath := filepath.Join(e.coverageOutputDir, "aggregate", "coverage-final.json") - aggCov, err := loadIstanbulJSON(aggregatePath) - if err != nil { - return fmt.Errorf("failed to load aggregate coverage: %w", err) - } - +func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate map[string]map[string]int) error { summary := CoverageSummary{ Timestamp: time.Now().Format(time.RFC3339), PerFile: make(map[string]CoverageFileSummary), @@ -461,30 +188,25 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord) error { totalCovered := 0 coveredFiles := 0 - for filePath, fileCov := range aggCov { - coverable := len(fileCov.S) + for filePath, lines := range aggregate { + coverable := len(lines) covered := 0 - for _, count := range fileCov.S { + for _, count := range lines { if count > 0 { covered++ } } - totalCoverable += coverable totalCovered += covered if covered > 0 { coveredFiles++ } - pct := 0.0 if coverable > 0 { pct = float64(covered) / float64(coverable) * 100 } - summary.PerFile[filePath] = CoverageFileSummary{ - CoveredLines: covered, - CoverableLines: coverable, - CoveragePct: pct, + CoveredLines: covered, CoverableLines: coverable, CoveragePct: pct, } } @@ -497,34 +219,20 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord) error { TotalCoverableLines: totalCoverable, TotalCoveredLines: totalCovered, CoveragePct: aggPct, - TotalFiles: len(aggCov), + TotalFiles: len(aggregate), CoveredFiles: coveredFiles, } - // Per-test summaries + // Per-test summaries from stored detail for _, record := range records { - testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(record.TestID)) - diffPath := filepath.Join(testDir, "coverage.json") - - testSummary := CoverageTestSummary{ - TestID: record.TestID, - TestName: record.TestName, - } - - data, err := os.ReadFile(diffPath) - if err == nil { - var diff map[string]CoverageFileDiff - if json.Unmarshal(data, &diff) == nil { - totalLines := 0 - for _, fd := range diff { - totalLines += fd.CoveredCount - } - testSummary.CoveredLines = totalLines - testSummary.FilesTouched = len(diff) + ts := CoverageTestSummary{TestID: record.TestID, TestName: record.TestName} + if detail := e.GetTestCoverageDetail(record.TestID); detail != nil { + for _, fd := range detail { + ts.CoveredLines += fd.CoveredCount } + ts.FilesTouched = len(detail) } - - summary.PerTest = append(summary.PerTest, testSummary) + summary.PerTest = append(summary.PerTest, ts) } // Write summary.json @@ -534,16 +242,14 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord) error { return err } - // Print to console + // Console output log.Stderrln(fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% (%d/%d coverable lines across %d files)", - aggPct, totalCovered, totalCoverable, len(aggCov))) + aggPct, totalCovered, totalCoverable, len(aggregate))) - // Print top files type fileStat struct { - path string - pct float64 - cov int - tot int + path string + pct float64 + cov, tot int } var stats []fileStat for fp, fs := range summary.PerFile { @@ -574,6 +280,25 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord) error { } log.Stderrln(fmt.Sprintf("\n Full report: %s", summaryPath)) - return nil } + +// --- Helpers --- + +func dedup(sorted []int) []int { + if len(sorted) == 0 { + return sorted + } + result := []int{sorted[0]} + for i := 1; i < len(sorted); i++ { + if sorted[i] != sorted[i-1] { + result = append(result, sorted[i]) + } + } + return result +} + +func sanitizeFileName(name string) string { + replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") + return replacer.Replace(name) +} diff --git a/internal/runner/executor.go b/internal/runner/executor.go index d45db04..e7dff82 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -95,9 +95,8 @@ type Executor struct { // Coverage coverageEnabled bool - coverageRawDir string // NODE_V8_COVERAGE output dir coveragePort int // Coverage snapshot server port - coverageOutputDir string // Final processed coverage output dir + coverageOutputDir string // Processed coverage output dir coveragePerTest map[string]map[string]CoverageFileDiff // testID -> per-file coverage diff coveragePerTestMu sync.Mutex } diff --git a/internal/runner/scripts/process-v8-coverage.js b/internal/runner/scripts/process-v8-coverage.js deleted file mode 100644 index d2c7953..0000000 --- a/internal/runner/scripts/process-v8-coverage.js +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env node -/** - * Processes raw V8 coverage files and outputs per-file statement counts. - * Usage: node process-v8-coverage.js [source-root] - * - * Outputs JSON: { "/path/to/file.js": { "statements": { "lineNumber": hitCount } } } - * Only includes user source files (excludes node_modules, node: builtins). - */ - -const fs = require('fs'); -const path = require('path'); - -const v8File = process.argv[2]; -const sourceRoot = process.argv[3] || process.cwd(); - -if (!v8File) { - console.error('Usage: node process-v8-coverage.js [source-root]'); - process.exit(1); -} - -const data = JSON.parse(fs.readFileSync(v8File, 'utf-8')); -const result = {}; - -for (const script of data.result) { - // Skip non-file URLs (node: builtins, eval, etc.) - if (!script.url.startsWith('file://')) continue; - - const filePath = script.url.replace('file://', ''); - - // Skip node_modules - if (filePath.includes('node_modules')) continue; - - // Skip files outside source root - if (!filePath.startsWith(sourceRoot)) continue; - - // Read source file to map byte offsets to line numbers - let source; - try { - source = fs.readFileSync(filePath, 'utf-8'); - } catch { - continue; - } - - // Build offset-to-line mapping - const lineStarts = [0]; - for (let i = 0; i < source.length; i++) { - if (source[i] === '\n') { - lineStarts.push(i + 1); - } - } - - function offsetToLine(offset) { - let lo = 0, hi = lineStarts.length - 1; - while (lo < hi) { - const mid = (lo + hi + 1) >> 1; - if (lineStarts[mid] <= offset) lo = mid; - else hi = mid - 1; - } - return lo + 1; // 1-based - } - - const lineCounts = {}; - - for (const func of script.functions) { - for (const range of func.ranges) { - if (range.count === 0) continue; - - const startLine = offsetToLine(range.startOffset); - const endLine = offsetToLine(range.endOffset); - - for (let line = startLine; line <= endLine; line++) { - lineCounts[line] = (lineCounts[line] || 0) + range.count; - } - } - } - - if (Object.keys(lineCounts).length > 0) { - result[filePath] = { lines: lineCounts }; - } -} - -process.stdout.write(JSON.stringify(result)); diff --git a/internal/runner/service.go b/internal/runner/service.go index ae633ec..9d003df 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -43,8 +43,23 @@ func (e *Executor) StartService() error { log.Debug("Starting service", "command", cfg.Service.Start.Command) - // Wrap command with fence sandboxing (if supported and enabled) command := cfg.Service.Start.Command + + // Coverage: set up output directory (env vars injected later, no command wrapping needed) + if e.coverageEnabled { + if e.coverageOutputDir == "" { + timestamp := time.Now().Format("20060102T150405") + e.coverageOutputDir = filepath.Join(".tusk", "coverage-"+timestamp) + } + if err := os.MkdirAll(e.coverageOutputDir, 0o750); err != nil { + return fmt.Errorf("failed to create coverage output dir: %w", err) + } + if e.coveragePort == 0 { + e.coveragePort = 19876 + } + } + + // Wrap command with fence sandboxing (if supported and enabled) replayOverridePath := e.getReplayComposeOverride() if replayOverridePath != "" && isComposeBasedStartCommand(command) { commandWithReplayOverride, injected, injectErr := injectComposeOverrideFile(command, replayOverridePath) @@ -147,26 +162,14 @@ func (e *Executor) StartService() error { env = append(env, "TUSK_DRIFT_MODE=REPLAY") - // Coverage: inject NODE_V8_COVERAGE and TUSK_COVERAGE_PORT + // Coverage: inject NODE_V8_COVERAGE dir and coverage port if e.coverageEnabled { - if e.coverageRawDir == "" { - timestamp := time.Now().Format("20060102T150405") - e.coverageRawDir = filepath.Join(".tusk", "coverage-raw-"+timestamp) - e.coverageOutputDir = filepath.Join(".tusk", "coverage-"+timestamp) - } - if err := os.MkdirAll(e.coverageRawDir, 0o750); err != nil { - return fmt.Errorf("failed to create coverage raw dir: %w", err) - } - if err := os.MkdirAll(e.coverageOutputDir, 0o750); err != nil { - return fmt.Errorf("failed to create coverage output dir: %w", err) - } - absCoverageRawDir, _ := filepath.Abs(e.coverageRawDir) - if e.coveragePort == 0 { - e.coveragePort = 19876 - } + coverageRawDir := filepath.Join(e.coverageOutputDir, ".v8-raw") + os.MkdirAll(coverageRawDir, 0o750) + absCoverageRawDir, _ := filepath.Abs(coverageRawDir) env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", absCoverageRawDir)) env = append(env, fmt.Sprintf("TUSK_COVERAGE_PORT=%d", e.coveragePort)) - log.Debug("Coverage enabled", "raw_dir", absCoverageRawDir, "port", e.coveragePort) + log.Debug("Coverage enabled", "v8_dir", absCoverageRawDir, "port", e.coveragePort) } e.serviceCmd.Env = env diff --git a/scripts/process-v8-coverage.js b/scripts/process-v8-coverage.js deleted file mode 100644 index d2c7953..0000000 --- a/scripts/process-v8-coverage.js +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env node -/** - * Processes raw V8 coverage files and outputs per-file statement counts. - * Usage: node process-v8-coverage.js [source-root] - * - * Outputs JSON: { "/path/to/file.js": { "statements": { "lineNumber": hitCount } } } - * Only includes user source files (excludes node_modules, node: builtins). - */ - -const fs = require('fs'); -const path = require('path'); - -const v8File = process.argv[2]; -const sourceRoot = process.argv[3] || process.cwd(); - -if (!v8File) { - console.error('Usage: node process-v8-coverage.js [source-root]'); - process.exit(1); -} - -const data = JSON.parse(fs.readFileSync(v8File, 'utf-8')); -const result = {}; - -for (const script of data.result) { - // Skip non-file URLs (node: builtins, eval, etc.) - if (!script.url.startsWith('file://')) continue; - - const filePath = script.url.replace('file://', ''); - - // Skip node_modules - if (filePath.includes('node_modules')) continue; - - // Skip files outside source root - if (!filePath.startsWith(sourceRoot)) continue; - - // Read source file to map byte offsets to line numbers - let source; - try { - source = fs.readFileSync(filePath, 'utf-8'); - } catch { - continue; - } - - // Build offset-to-line mapping - const lineStarts = [0]; - for (let i = 0; i < source.length; i++) { - if (source[i] === '\n') { - lineStarts.push(i + 1); - } - } - - function offsetToLine(offset) { - let lo = 0, hi = lineStarts.length - 1; - while (lo < hi) { - const mid = (lo + hi + 1) >> 1; - if (lineStarts[mid] <= offset) lo = mid; - else hi = mid - 1; - } - return lo + 1; // 1-based - } - - const lineCounts = {}; - - for (const func of script.functions) { - for (const range of func.ranges) { - if (range.count === 0) continue; - - const startLine = offsetToLine(range.startOffset); - const endLine = offsetToLine(range.endOffset); - - for (let line = startLine; line <= endLine; line++) { - lineCounts[line] = (lineCounts[line] || 0) + range.count; - } - } - } - - if (Object.keys(lineCounts).length > 0) { - result[filePath] = { lines: lineCounts }; - } -} - -process.stdout.write(JSON.stringify(result)); From 6d8b56966f3fb8602ba7a3fcab6d7f6a7da5244e Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 01:16:34 -0700 Subject: [PATCH 04/36] feat: add baseline snapshot for coverage denominator CLI calls /snapshot?baseline=true at startup to capture all coverable lines (including uncovered at count=0). This baseline is used as the starting point for aggregate coverage, providing an accurate denominator. Added TakeCoverageBaseline() and mergeWithBaseline() to separate baseline (denominator) from per-test (delta) snapshots. --- cmd/run.go | 11 ++++++---- internal/runner/coverage.go | 41 ++++++++++++++++++++++++++++++++----- internal/runner/executor.go | 13 ++++++++++-- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index fd0deec..ee55766 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -875,14 +875,17 @@ func runTests(cmd *cobra.Command, args []string) error { log.Stderrln(fmt.Sprintf("➀ Running %d tests (concurrency: %d)...\n", len(tests), executor.GetConcurrency())) } - // Coverage: take baseline snapshot to discard startup coverage. - // v8.takeCoverage() resets counters, so the first real test gets clean data. + // Coverage: take baseline with ?baseline=true to capture ALL coverable lines + // (including uncovered at count=0) for the aggregate denominator. + // This also resets counters so the first test gets clean data. if coverageEnabled { time.Sleep(500 * time.Millisecond) // let coverage server start - if _, err := executor.TakeCoverageSnapshot(); err != nil { + baseline, err := executor.TakeCoverageBaseline() + if err != nil { log.Warn("Failed to take baseline coverage snapshot", "error", err) } else { - log.Debug("Coverage baseline taken (startup coverage discarded, counters reset)") + executor.SetCoverageBaseline(baseline) + log.Debug("Coverage baseline taken (counters reset, all coverable lines captured)") } } diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 1ad634a..3ef7092 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -21,12 +21,28 @@ const coverageSnapshotTimeout = 5 * time.Second // processes the coverage data, and returns per-file line counts. // V8's takeCoverage() auto-resets counters, so each call returns ONLY the // coverage since the last call - true per-test coverage with no diffing needed. +// TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. +// Returns per-file line counts for this test only (counters auto-reset). func (e *Executor) TakeCoverageSnapshot() (map[string]map[string]int, error) { + return e.callCoverageEndpoint(false) +} + +// TakeCoverageBaseline calls the SDK's coverage snapshot endpoint with ?baseline=true. +// Returns ALL coverable lines (including uncovered at count=0) for the aggregate denominator. +// Also resets counters so the first real test gets clean data. +func (e *Executor) TakeCoverageBaseline() (map[string]map[string]int, error) { + return e.callCoverageEndpoint(true) +} + +func (e *Executor) callCoverageEndpoint(baseline bool) (map[string]map[string]int, error) { if !e.coverageEnabled || e.coveragePort == 0 { return nil, nil } url := fmt.Sprintf("http://127.0.0.1:%d/snapshot", e.coveragePort) + if baseline { + url += "?baseline=true" + } httpClient := &http.Client{Timeout: coverageSnapshotTimeout} resp, err := httpClient.Get(url) @@ -124,17 +140,31 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { } } - // Compute aggregate by merging all per-test coverage (union of covered lines) - aggregate := mergeLineCounts(records) + // Compute aggregate: start with baseline (all coverable lines including count=0), + // then merge in per-test coverage. This gives accurate denominator. + aggregate := mergeWithBaseline(e.coverageBaseline, records) // Print and write summary return e.printCoverageSummary(records, aggregate) } -// mergeLineCounts unions all per-test line counts into an aggregate. -// A line's count in the aggregate = sum of counts across all tests. -func mergeLineCounts(records []CoverageTestRecord) map[string]map[string]int { +// mergeWithBaseline creates aggregate coverage starting from the baseline +// (which has ALL coverable lines including count=0), then merging in per-test data. +// If no baseline is available, falls back to merging per-test data only. +func mergeWithBaseline(baseline map[string]map[string]int, records []CoverageTestRecord) map[string]map[string]int { merged := make(map[string]map[string]int) + + // Start with baseline (all coverable lines, count=0 for uncovered) + if baseline != nil { + for filePath, lines := range baseline { + merged[filePath] = make(map[string]int) + for line, count := range lines { + merged[filePath][line] = count + } + } + } + + // Merge in per-test coverage (add counts) for _, record := range records { for filePath, lines := range record.LineCounts { if merged[filePath] == nil { @@ -145,6 +175,7 @@ func mergeLineCounts(records []CoverageTestRecord) map[string]map[string]int { } } } + return merged } diff --git a/internal/runner/executor.go b/internal/runner/executor.go index e7dff82..0067eba 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -97,8 +97,9 @@ type Executor struct { coverageEnabled bool coveragePort int // Coverage snapshot server port coverageOutputDir string // Processed coverage output dir - coveragePerTest map[string]map[string]CoverageFileDiff // testID -> per-file coverage diff - coveragePerTestMu sync.Mutex + coveragePerTest map[string]map[string]CoverageFileDiff // testID -> per-file coverage diff + coveragePerTestMu sync.Mutex + coverageBaseline map[string]map[string]int // baseline: all coverable lines (including count=0) } func NewExecutor() *Executor { @@ -494,6 +495,14 @@ func (e *Executor) GetCoverageOutputDir() string { return e.coverageOutputDir } +func (e *Executor) SetCoverageBaseline(baseline map[string]map[string]int) { + e.coverageBaseline = baseline +} + +func (e *Executor) GetCoverageBaseline() map[string]map[string]int { + return e.coverageBaseline +} + // SetTestCoverageDetail stores per-test coverage diff for display in TUI/print. func (e *Executor) SetTestCoverageDetail(testID string, detail map[string]CoverageFileDiff) { e.coveragePerTestMu.Lock() From c4812cb6f39b461763bbd31894fbcbbe1b8eccfa Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 01:22:39 -0700 Subject: [PATCH 05/36] chore: code review cleanup + unit tests for coverage - Replace fmt.Sscanf with strconv.Atoi (idiomatic Go) - Language-agnostic comments (remove Node-specific references) - Add 22 unit tests for coverage pure functions: sanitizeFileName, dedup, LinecountsToCoverageDetail, mergeWithBaseline --- cmd/run.go | 2 +- internal/runner/coverage.go | 14 +- internal/runner/coverage_test.go | 229 +++++++++++++++++++++++++++++++ internal/runner/service.go | 6 +- 4 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 internal/runner/coverage_test.go diff --git a/cmd/run.go b/cmd/run.go index ee55766..2ae5298 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -409,7 +409,7 @@ func runTests(cmd *cobra.Command, args []string) error { existingCallback(res, test) } - // Take coverage snapshot. V8's takeCoverage() auto-resets counters, + // Take coverage snapshot. The SDK resets counters on each call, // so the response contains ONLY coverage from this test. lineCounts, err := executor.TakeCoverageSnapshot() if err != nil { diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 3ef7092..5e6abaa 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -16,11 +17,6 @@ import ( const coverageSnapshotTimeout = 5 * time.Second -// TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. -// The SDK calls v8.takeCoverage() (Node) or coverage.py stop/start (Python), -// processes the coverage data, and returns per-file line counts. -// V8's takeCoverage() auto-resets counters, so each call returns ONLY the -// coverage since the last call - true per-test coverage with no diffing needed. // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. // Returns per-file line counts for this test only (counters auto-reset). func (e *Executor) TakeCoverageSnapshot() (map[string]map[string]int, error) { @@ -96,11 +92,11 @@ func LinecountsToCoverageDetail(lineCounts map[string]map[string]int) map[string var covered []int for lineStr, count := range lines { if count > 0 { - line := 0 - fmt.Sscanf(lineStr, "%d", &line) - if line > 0 { - covered = append(covered, line) + line, err := strconv.Atoi(lineStr) + if err != nil || line <= 0 { + continue } + covered = append(covered, line) } } if len(covered) > 0 { diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go new file mode 100644 index 0000000..1e0622d --- /dev/null +++ b/internal/runner/coverage_test.go @@ -0,0 +1,229 @@ +package runner + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSanitizeFileName(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {name: "no special chars", input: "simple", expected: "simple"}, + {name: "forward slashes", input: "a/b/c", expected: "a_b_c"}, + {name: "backslashes", input: "a\\b\\c", expected: "a_b_c"}, + {name: "colons", input: "a:b:c", expected: "a_b_c"}, + {name: "spaces", input: "a b c", expected: "a_b_c"}, + {name: "mixed separators", input: "path/to:some file\\here", expected: "path_to_some_file_here"}, + {name: "empty string", input: "", expected: ""}, + {name: "already clean", input: "test_id_123", expected: "test_id_123"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := sanitizeFileName(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestDedup(t *testing.T) { + tests := []struct { + name string + input []int + expected []int + }{ + {name: "empty", input: []int{}, expected: []int{}}, + {name: "single element", input: []int{1}, expected: []int{1}}, + {name: "no duplicates", input: []int{1, 2, 3}, expected: []int{1, 2, 3}}, + {name: "all duplicates", input: []int{5, 5, 5}, expected: []int{5}}, + {name: "some duplicates", input: []int{1, 1, 2, 3, 3, 4}, expected: []int{1, 2, 3, 4}}, + {name: "duplicates at end", input: []int{1, 2, 3, 3}, expected: []int{1, 2, 3}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := dedup(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestLinecountsToCoverageDetail(t *testing.T) { + t.Run("empty input", func(t *testing.T) { + result := LinecountsToCoverageDetail(nil) + assert.Empty(t, result) + }) + + t.Run("empty map", func(t *testing.T) { + result := LinecountsToCoverageDetail(map[string]map[string]int{}) + assert.Empty(t, result) + }) + + t.Run("single file with covered lines", func(t *testing.T) { + input := map[string]map[string]int{ + "/app/main.go": { + "1": 1, + "2": 3, + "5": 0, + "10": 1, + }, + } + result := LinecountsToCoverageDetail(input) + require.Contains(t, result, "/app/main.go") + + fd := result["/app/main.go"] + assert.Equal(t, []int{1, 2, 10}, fd.CoveredLines) + assert.Equal(t, 3, fd.CoveredCount) + assert.Equal(t, 4, fd.CoverableLines) // total lines in the map + }) + + t.Run("file with only zero counts is excluded", func(t *testing.T) { + input := map[string]map[string]int{ + "/app/unused.go": { + "1": 0, + "2": 0, + }, + } + result := LinecountsToCoverageDetail(input) + assert.Empty(t, result) + }) + + t.Run("invalid line number strings are skipped", func(t *testing.T) { + input := map[string]map[string]int{ + "/app/main.go": { + "abc": 1, + "0": 1, // line 0 is invalid (1-based) + "-1": 1, // negative line is invalid + "5": 1, // valid + }, + } + result := LinecountsToCoverageDetail(input) + require.Contains(t, result, "/app/main.go") + fd := result["/app/main.go"] + assert.Equal(t, []int{5}, fd.CoveredLines) + assert.Equal(t, 1, fd.CoveredCount) + }) + + t.Run("multiple files", func(t *testing.T) { + input := map[string]map[string]int{ + "/app/a.go": {"1": 1, "2": 1}, + "/app/b.go": {"10": 2, "20": 0}, + } + result := LinecountsToCoverageDetail(input) + assert.Len(t, result, 2) + assert.Equal(t, 2, result["/app/a.go"].CoveredCount) + assert.Equal(t, 1, result["/app/b.go"].CoveredCount) + }) + + t.Run("covered lines are sorted and deduped", func(t *testing.T) { + input := map[string]map[string]int{ + "/app/main.go": { + "10": 1, + "3": 1, + "7": 1, + "1": 1, + }, + } + result := LinecountsToCoverageDetail(input) + require.Contains(t, result, "/app/main.go") + assert.Equal(t, []int{1, 3, 7, 10}, result["/app/main.go"].CoveredLines) + }) +} + +func TestMergeWithBaseline(t *testing.T) { + t.Run("nil baseline nil records", func(t *testing.T) { + result := mergeWithBaseline(nil, nil) + assert.Empty(t, result) + }) + + t.Run("nil baseline with records", func(t *testing.T) { + records := []CoverageTestRecord{ + { + TestID: "test-1", + LineCounts: map[string]map[string]int{ + "/app/main.go": {"1": 1, "2": 3}, + }, + }, + } + result := mergeWithBaseline(nil, records) + require.Contains(t, result, "/app/main.go") + assert.Equal(t, 1, result["/app/main.go"]["1"]) + assert.Equal(t, 3, result["/app/main.go"]["2"]) + }) + + t.Run("baseline with no records", func(t *testing.T) { + baseline := map[string]map[string]int{ + "/app/main.go": {"1": 0, "2": 0, "3": 0}, + } + result := mergeWithBaseline(baseline, nil) + require.Contains(t, result, "/app/main.go") + assert.Equal(t, 0, result["/app/main.go"]["1"]) + assert.Equal(t, 0, result["/app/main.go"]["2"]) + assert.Equal(t, 0, result["/app/main.go"]["3"]) + }) + + t.Run("baseline merged with records adds counts", func(t *testing.T) { + baseline := map[string]map[string]int{ + "/app/main.go": {"1": 0, "2": 0, "3": 0, "4": 0}, + } + records := []CoverageTestRecord{ + { + TestID: "test-1", + LineCounts: map[string]map[string]int{ + "/app/main.go": {"1": 1, "3": 2}, + }, + }, + { + TestID: "test-2", + LineCounts: map[string]map[string]int{ + "/app/main.go": {"1": 1, "4": 1}, + }, + }, + } + result := mergeWithBaseline(baseline, records) + require.Contains(t, result, "/app/main.go") + assert.Equal(t, 2, result["/app/main.go"]["1"]) // 0 + 1 + 1 + assert.Equal(t, 0, result["/app/main.go"]["2"]) // baseline 0, no test coverage + assert.Equal(t, 2, result["/app/main.go"]["3"]) // 0 + 2 + assert.Equal(t, 1, result["/app/main.go"]["4"]) // 0 + 1 + }) + + t.Run("records can add new files not in baseline", func(t *testing.T) { + baseline := map[string]map[string]int{ + "/app/main.go": {"1": 0}, + } + records := []CoverageTestRecord{ + { + TestID: "test-1", + LineCounts: map[string]map[string]int{ + "/app/new_file.go": {"10": 5}, + }, + }, + } + result := mergeWithBaseline(baseline, records) + assert.Len(t, result, 2) + assert.Equal(t, 5, result["/app/new_file.go"]["10"]) + }) + + t.Run("baseline is not mutated", func(t *testing.T) { + baseline := map[string]map[string]int{ + "/app/main.go": {"1": 0}, + } + records := []CoverageTestRecord{ + { + TestID: "test-1", + LineCounts: map[string]map[string]int{ + "/app/main.go": {"1": 5}, + }, + }, + } + _ = mergeWithBaseline(baseline, records) + // Original baseline should be untouched + assert.Equal(t, 0, baseline["/app/main.go"]["1"]) + }) +} diff --git a/internal/runner/service.go b/internal/runner/service.go index 9d003df..04544b9 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -162,14 +162,16 @@ func (e *Executor) StartService() error { env = append(env, "TUSK_DRIFT_MODE=REPLAY") - // Coverage: inject NODE_V8_COVERAGE dir and coverage port + // Coverage: inject env vars that SDK coverage servers listen for. + // NODE_V8_COVERAGE is required by the Node SDK to enable V8 coverage collection. + // TUSK_COVERAGE_PORT tells both Node and Python SDKs which port to serve snapshots on. if e.coverageEnabled { coverageRawDir := filepath.Join(e.coverageOutputDir, ".v8-raw") os.MkdirAll(coverageRawDir, 0o750) absCoverageRawDir, _ := filepath.Abs(coverageRawDir) env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", absCoverageRawDir)) env = append(env, fmt.Sprintf("TUSK_COVERAGE_PORT=%d", e.coveragePort)) - log.Debug("Coverage enabled", "v8_dir", absCoverageRawDir, "port", e.coveragePort) + log.Debug("Coverage enabled", "raw_dir", absCoverageRawDir, "port", e.coveragePort) } e.serviceCmd.Env = env From 224f16aefd3a887abeaa8580b76a5a67485702be Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 12:01:53 -0700 Subject: [PATCH 06/36] fix: add baseline snapshot to environment-based replay path The baseline snapshot (?baseline=true) was only taken in the single-env fallback path but tests run through ReplayTestsByEnvironment. Added baseline capture after StartEnvironment() in environment_replay.go. Also removed debug prints from coverage.go. --- internal/runner/environment_replay.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/runner/environment_replay.go b/internal/runner/environment_replay.go index 15e121c..9475228 100644 --- a/internal/runner/environment_replay.go +++ b/internal/runner/environment_replay.go @@ -62,6 +62,18 @@ func ReplayTestsByEnvironment( log.ServiceLog(fmt.Sprintf("βœ“ Environment ready (%.1fs)", envStartDuration)) log.Stderrln(fmt.Sprintf("βœ“ Environment ready (%.1fs)", envStartDuration)) + // Coverage: take baseline snapshot to capture all coverable lines and reset counters + if executor.IsCoverageEnabled() { + time.Sleep(500 * time.Millisecond) // let coverage server start + baseline, err := executor.TakeCoverageBaseline() + if err != nil { + log.Warn("Failed to take baseline coverage snapshot", "error", err) + } else { + executor.SetCoverageBaseline(baseline) + log.Debug("Coverage baseline taken (counters reset, all coverable lines captured)") + } + } + // 3. Run tests for this environment results, err := executor.RunTests(group.Tests) if err != nil { From d588ca732417ff3e954ca7aebdf3029f8ae3c72e Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 15:29:25 -0700 Subject: [PATCH 07/36] fix: enforce concurrency=1, retry baseline, normalize file paths - Force concurrency=1 when --coverage is enabled (per-test snapshots require serial execution) - Replace hardcoded 500ms sleep with retry loop (15 attempts, 200ms apart) for baseline snapshot - handles slow SDK startup - Normalize absolute file paths to repo-relative (using cwd as base) so paths are consistent across machines for backend storage - Accumulate baselines across environment groups (merge, don't overwrite) so coverage is correct when service restarts between groups --- cmd/run.go | 6 +++-- internal/runner/coverage.go | 35 +++++++++++++++++++++++++-- internal/runner/environment_replay.go | 1 - internal/runner/executor.go | 17 ++++++++++++- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 2ae5298..8cc68e3 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -311,7 +311,10 @@ func runTests(cmd *cobra.Command, args []string) error { executor.SetEnableServiceLogs(enableServiceLogs || debug) if coverageEnabled { executor.SetCoverageEnabled(true) - log.Stderrln("➀ Coverage collection enabled") + // Coverage requires serial execution (concurrency=1) because per-test + // snapshots rely on the SDK resetting counters between tests. + executor.SetConcurrency(1) + log.Stderrln("➀ Coverage collection enabled (concurrency forced to 1)") } if saveResults { if resultsDir == "" { @@ -879,7 +882,6 @@ func runTests(cmd *cobra.Command, args []string) error { // (including uncovered at count=0) for the aggregate denominator. // This also resets counters so the first test gets clean data. if coverageEnabled { - time.Sleep(500 * time.Millisecond) // let coverage server start baseline, err := executor.TakeCoverageBaseline() if err != nil { log.Warn("Failed to take baseline coverage snapshot", "error", err) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 5e6abaa..5190c11 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -26,8 +26,18 @@ func (e *Executor) TakeCoverageSnapshot() (map[string]map[string]int, error) { // TakeCoverageBaseline calls the SDK's coverage snapshot endpoint with ?baseline=true. // Returns ALL coverable lines (including uncovered at count=0) for the aggregate denominator. // Also resets counters so the first real test gets clean data. +// Retries with backoff since the coverage server may not be ready immediately after service start. func (e *Executor) TakeCoverageBaseline() (map[string]map[string]int, error) { - return e.callCoverageEndpoint(true) + var lastErr error + for attempt := 0; attempt < 15; attempt++ { + result, err := e.callCoverageEndpoint(true) + if err == nil { + return result, nil + } + lastErr = err + time.Sleep(200 * time.Millisecond) + } + return nil, fmt.Errorf("coverage baseline failed after retries: %w", lastErr) } func (e *Executor) callCoverageEndpoint(baseline bool) (map[string]map[string]int, error) { @@ -68,7 +78,8 @@ func (e *Executor) callCoverageEndpoint(baseline bool) (map[string]map[string]in return nil, fmt.Errorf("coverage snapshot returned ok=false") } - return result.Coverage, nil + // Normalize absolute paths to repo-relative paths + return normalizeFilePaths(result.Coverage), nil } // CoverageTestRecord holds per-test coverage data. @@ -329,3 +340,23 @@ func sanitizeFileName(name string) string { replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") return replacer.Replace(name) } + +// normalizeFilePaths converts absolute file paths to repo-relative paths. +// Uses the current working directory as the base (which is the project root +// where .tusk/config.yaml lives). This ensures consistent paths across machines. +func normalizeFilePaths(lineCounts map[string]map[string]int) map[string]map[string]int { + cwd, err := os.Getwd() + if err != nil { + return lineCounts + } + + normalized := make(map[string]map[string]int, len(lineCounts)) + for absPath, lines := range lineCounts { + relPath, err := filepath.Rel(cwd, absPath) + if err != nil { + relPath = absPath // keep absolute if rel fails + } + normalized[relPath] = lines + } + return normalized +} diff --git a/internal/runner/environment_replay.go b/internal/runner/environment_replay.go index 9475228..ebf814f 100644 --- a/internal/runner/environment_replay.go +++ b/internal/runner/environment_replay.go @@ -64,7 +64,6 @@ func ReplayTestsByEnvironment( // Coverage: take baseline snapshot to capture all coverable lines and reset counters if executor.IsCoverageEnabled() { - time.Sleep(500 * time.Millisecond) // let coverage server start baseline, err := executor.TakeCoverageBaseline() if err != nil { log.Warn("Failed to take baseline coverage snapshot", "error", err) diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 0067eba..5c12497 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -495,8 +495,23 @@ func (e *Executor) GetCoverageOutputDir() string { return e.coverageOutputDir } +// SetCoverageBaseline merges new baseline data into the existing baseline. +// Called per environment group - accumulates across service restarts. func (e *Executor) SetCoverageBaseline(baseline map[string]map[string]int) { - e.coverageBaseline = baseline + if e.coverageBaseline == nil { + e.coverageBaseline = make(map[string]map[string]int) + } + for filePath, lines := range baseline { + if e.coverageBaseline[filePath] == nil { + e.coverageBaseline[filePath] = make(map[string]int) + } + for line, count := range lines { + // Keep the existing count if it's already tracked (don't overwrite covered with uncovered) + if existing, ok := e.coverageBaseline[filePath][line]; !ok || existing == 0 { + e.coverageBaseline[filePath][line] = count + } + } + } } func (e *Executor) GetCoverageBaseline() map[string]map[string]int { From 6c3a212f70e640f7287da3915e459b62e53786bb Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 15:31:04 -0700 Subject: [PATCH 08/36] fix: use git root for file path normalization Use git rev-parse --show-toplevel as the base for relative paths instead of cwd. This correctly handles monorepo files outside the service directory (e.g., ../shared/utils.js becomes shared/utils.js). Falls back to cwd if not in a git repo. Paths outside the git root are kept absolute. --- internal/runner/coverage.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 5190c11..ad06d22 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "os/exec" "path/filepath" "sort" "strconv" @@ -342,21 +343,37 @@ func sanitizeFileName(name string) string { } // normalizeFilePaths converts absolute file paths to repo-relative paths. -// Uses the current working directory as the base (which is the project root -// where .tusk/config.yaml lives). This ensures consistent paths across machines. +// Uses git root as the base (consistent across machines, handles monorepo +// files outside the service directory like ../shared/utils.js). +// Falls back to cwd if not in a git repo. func normalizeFilePaths(lineCounts map[string]map[string]int) map[string]map[string]int { - cwd, err := os.Getwd() - if err != nil { + base := getPathNormalizationBase() + if base == "" { return lineCounts } normalized := make(map[string]map[string]int, len(lineCounts)) for absPath, lines := range lineCounts { - relPath, err := filepath.Rel(cwd, absPath) - if err != nil { - relPath = absPath // keep absolute if rel fails + relPath, err := filepath.Rel(base, absPath) + if err != nil || strings.HasPrefix(relPath, "..") { + // Outside the base - keep as-is rather than producing ../../... paths + relPath = absPath } normalized[relPath] = lines } return normalized } + +// getPathNormalizationBase returns the git root, falling back to cwd. +func getPathNormalizationBase() string { + // Try git root first (handles monorepo files outside service dir) + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + if out, err := cmd.Output(); err == nil { + return strings.TrimSpace(string(out)) + } + // Fallback to cwd + if cwd, err := os.Getwd(); err == nil { + return cwd + } + return "" +} From 27f76b66f36074d4353f709fa6c3f90385ee555d Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 15:36:22 -0700 Subject: [PATCH 09/36] refactor: extract GetGitRootDir to shared utils package Move git root detection from onboard-cloud/helpers.go to utils/filesystem.go as GetGitRootDir(). Reuse in coverage.go for file path normalization. Removes duplicate exec.Command("git", "rev-parse", "--show-toplevel") calls. --- internal/runner/coverage.go | 9 +++------ internal/tui/onboard-cloud/helpers.go | 7 +------ internal/utils/filesystem.go | 12 ++++++++++++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index ad06d22..d874925 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "os" - "os/exec" "path/filepath" "sort" "strconv" @@ -14,6 +13,7 @@ import ( "time" "github.com/Use-Tusk/tusk-cli/internal/log" + "github.com/Use-Tusk/tusk-cli/internal/utils" ) const coverageSnapshotTimeout = 5 * time.Second @@ -366,12 +366,9 @@ func normalizeFilePaths(lineCounts map[string]map[string]int) map[string]map[str // getPathNormalizationBase returns the git root, falling back to cwd. func getPathNormalizationBase() string { - // Try git root first (handles monorepo files outside service dir) - cmd := exec.Command("git", "rev-parse", "--show-toplevel") - if out, err := cmd.Output(); err == nil { - return strings.TrimSpace(string(out)) + if root, err := utils.GetGitRootDir(); err == nil { + return root } - // Fallback to cwd if cwd, err := os.Getwd(); err == nil { return cwd } diff --git a/internal/tui/onboard-cloud/helpers.go b/internal/tui/onboard-cloud/helpers.go index 3b80e77..ffcdab1 100644 --- a/internal/tui/onboard-cloud/helpers.go +++ b/internal/tui/onboard-cloud/helpers.go @@ -223,12 +223,7 @@ func saveSelectedClientToCLIConfig(clientID, clientName string) { } func getGitRootDir() (string, error) { - cmd := exec.Command("git", "rev-parse", "--show-toplevel") - out, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("failed to get git root: %w", err) - } - return strings.TrimSpace(string(out)), nil + return utils.GetGitRootDir() } func detectGitHubIndicators() bool { diff --git a/internal/utils/filesystem.go b/internal/utils/filesystem.go index 30b2f59..2336b59 100644 --- a/internal/utils/filesystem.go +++ b/internal/utils/filesystem.go @@ -3,6 +3,7 @@ package utils import ( "fmt" "os" + "os/exec" "path/filepath" "strings" ) @@ -138,6 +139,17 @@ func EnsureDir(dir string) error { return os.MkdirAll(dir, 0o750) } +// GetGitRootDir returns the root of the current git repository. +// Returns empty string and an error if not in a git repo. +func GetGitRootDir() (string, error) { + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get git root: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + // FindTraceFile searches for a JSONL trace file containing the given trace ID. // If filename is provided, it tries that first before searching func FindTraceFile(traceID string, filename string) (string, error) { From 1b15d280467f2fe6786f106baecd6fb7cf020d0a Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 16:03:28 -0700 Subject: [PATCH 10/36] refactor: remove file output, keep coverage data in memory only - Remove per-test coverage.json file writing - Remove summary.json file writing - Remove coverageOutputDir from executor (no longer needed) - Use os.TempDir() for NODE_V8_COVERAGE instead of .tusk/coverage-*/.v8-raw/ - No files left in user's project after coverage run - All data stays in memory: printed to console, ready for backend upload --- internal/runner/coverage.go | 29 +++-------------------------- internal/runner/executor.go | 13 ++++--------- internal/runner/service.go | 27 ++++++++++----------------- 3 files changed, 17 insertions(+), 52 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index d874925..d40316c 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -124,7 +124,8 @@ func LinecountsToCoverageDetail(lineCounts map[string]map[string]int) map[string return result } -// ProcessCoverage writes per-test coverage files and prints the summary. +// ProcessCoverage computes aggregate coverage and prints the summary. +// All data stays in memory - no files written to user's project. func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { if !e.coverageEnabled || len(records) == 0 { return nil @@ -132,27 +133,11 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { log.Stderrln("\n➀ Processing coverage data...") - // Write per-test coverage files - for _, record := range records { - detail := e.GetTestCoverageDetail(record.TestID) - if detail == nil { - continue - } - testDir := filepath.Join(e.coverageOutputDir, sanitizeFileName(record.TestID)) - if err := os.MkdirAll(testDir, 0o750); err != nil { - return err - } - data, _ := json.MarshalIndent(detail, "", " ") - if err := os.WriteFile(filepath.Join(testDir, "coverage.json"), data, 0o644); err != nil { - return err - } - } - // Compute aggregate: start with baseline (all coverable lines including count=0), // then merge in per-test coverage. This gives accurate denominator. aggregate := mergeWithBaseline(e.coverageBaseline, records) - // Print and write summary + // Print summary to console return e.printCoverageSummary(records, aggregate) } @@ -274,13 +259,6 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate summary.PerTest = append(summary.PerTest, ts) } - // Write summary.json - summaryPath := filepath.Join(e.coverageOutputDir, "summary.json") - summaryData, _ := json.MarshalIndent(summary, "", " ") - if err := os.WriteFile(summaryPath, summaryData, 0o644); err != nil { - return err - } - // Console output log.Stderrln(fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% (%d/%d coverable lines across %d files)", aggPct, totalCovered, totalCoverable, len(aggregate))) @@ -318,7 +296,6 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate log.Stderrln(fmt.Sprintf(" %-40s %d lines across %d files", name, ts.CoveredLines, ts.FilesTouched)) } - log.Stderrln(fmt.Sprintf("\n Full report: %s", summaryPath)) return nil } diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 5c12497..3d3387c 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -94,12 +94,11 @@ type Executor struct { replaySandboxConfigPath string // Coverage - coverageEnabled bool - coveragePort int // Coverage snapshot server port - coverageOutputDir string // Processed coverage output dir - coveragePerTest map[string]map[string]CoverageFileDiff // testID -> per-file coverage diff + coverageEnabled bool + coveragePort int // Coverage snapshot server port + coveragePerTest map[string]map[string]CoverageFileDiff coveragePerTestMu sync.Mutex - coverageBaseline map[string]map[string]int // baseline: all coverable lines (including count=0) + coverageBaseline map[string]map[string]int } func NewExecutor() *Executor { @@ -491,10 +490,6 @@ func (e *Executor) IsCoverageEnabled() bool { return e.coverageEnabled } -func (e *Executor) GetCoverageOutputDir() string { - return e.coverageOutputDir -} - // SetCoverageBaseline merges new baseline data into the existing baseline. // Called per environment group - accumulates across service restarts. func (e *Executor) SetCoverageBaseline(baseline map[string]map[string]int) { diff --git a/internal/runner/service.go b/internal/runner/service.go index 04544b9..46ae227 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -45,18 +45,9 @@ func (e *Executor) StartService() error { command := cfg.Service.Start.Command - // Coverage: set up output directory (env vars injected later, no command wrapping needed) - if e.coverageEnabled { - if e.coverageOutputDir == "" { - timestamp := time.Now().Format("20060102T150405") - e.coverageOutputDir = filepath.Join(".tusk", "coverage-"+timestamp) - } - if err := os.MkdirAll(e.coverageOutputDir, 0o750); err != nil { - return fmt.Errorf("failed to create coverage output dir: %w", err) - } - if e.coveragePort == 0 { - e.coveragePort = 19876 - } + // Coverage: set port (env vars injected after sandbox wrapping) + if e.coverageEnabled && e.coveragePort == 0 { + e.coveragePort = 19876 } // Wrap command with fence sandboxing (if supported and enabled) @@ -166,12 +157,14 @@ func (e *Executor) StartService() error { // NODE_V8_COVERAGE is required by the Node SDK to enable V8 coverage collection. // TUSK_COVERAGE_PORT tells both Node and Python SDKs which port to serve snapshots on. if e.coverageEnabled { - coverageRawDir := filepath.Join(e.coverageOutputDir, ".v8-raw") - os.MkdirAll(coverageRawDir, 0o750) - absCoverageRawDir, _ := filepath.Abs(coverageRawDir) - env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", absCoverageRawDir)) + // Use temp dir for V8 coverage files (SDK reads + deletes immediately, nothing persists) + v8CoverageDir, err := os.MkdirTemp("", "tusk-v8-coverage-*") + if err != nil { + return fmt.Errorf("failed to create temp dir for V8 coverage: %w", err) + } + env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", v8CoverageDir)) env = append(env, fmt.Sprintf("TUSK_COVERAGE_PORT=%d", e.coveragePort)) - log.Debug("Coverage enabled", "raw_dir", absCoverageRawDir, "port", e.coveragePort) + log.Debug("Coverage enabled", "v8_dir", v8CoverageDir, "port", e.coveragePort) } e.serviceCmd.Env = env From 0be4ab345533e55134c486903d02210a106eeba1 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 17:33:41 -0700 Subject: [PATCH 11/36] feat: add branch coverage tracking to CLI - New types: BranchInfo, FileCoverageData, CoverageSnapshot - Parse branch data from SDK response (totalBranches, coveredBranches, per-line detail) - Display branch coverage in aggregate: "85.5% lines, 93.3% branches" - Merge branch data across baseline and per-test snapshots - Update CoverageFileDiff, CoverageFileSummary, CoverageAggregate with branch fields - Update all tests for new type structure --- cmd/run.go | 8 +- internal/runner/coverage.go | 214 ++++++++++++++++++++++--------- internal/runner/coverage_test.go | 191 ++++++++++++--------------- internal/runner/executor.go | 41 +++--- 4 files changed, 269 insertions(+), 185 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 8cc68e3..616cde8 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -422,14 +422,14 @@ func runTests(cmd *cobra.Command, args []string) error { coverageMu.Lock() coverageRecords = append(coverageRecords, runner.CoverageTestRecord{ - TestID: test.TraceID, - TestName: fmt.Sprintf("%s %s", test.Method, test.Path), - LineCounts: lineCounts, + TestID: test.TraceID, + TestName: fmt.Sprintf("%s %s", test.Method, test.Path), + Coverage: lineCounts, }) coverageMu.Unlock() // Store detail for TUI display - detail := runner.LinecountsToCoverageDetail(lineCounts) + detail := runner.SnapshotToCoverageDetail(lineCounts) executor.SetTestCoverageDetail(test.TraceID, detail) // Print sub-line in --print mode diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index d40316c..57f6271 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -19,16 +19,15 @@ import ( const coverageSnapshotTimeout = 5 * time.Second // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. -// Returns per-file line counts for this test only (counters auto-reset). -func (e *Executor) TakeCoverageSnapshot() (map[string]map[string]int, error) { +// Returns per-file coverage data for this test only (counters auto-reset). +func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { return e.callCoverageEndpoint(false) } // TakeCoverageBaseline calls the SDK's coverage snapshot endpoint with ?baseline=true. // Returns ALL coverable lines (including uncovered at count=0) for the aggregate denominator. -// Also resets counters so the first real test gets clean data. // Retries with backoff since the coverage server may not be ready immediately after service start. -func (e *Executor) TakeCoverageBaseline() (map[string]map[string]int, error) { +func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { var lastErr error for attempt := 0; attempt < 15; attempt++ { result, err := e.callCoverageEndpoint(true) @@ -41,7 +40,7 @@ func (e *Executor) TakeCoverageBaseline() (map[string]map[string]int, error) { return nil, fmt.Errorf("coverage baseline failed after retries: %w", lastErr) } -func (e *Executor) callCoverageEndpoint(baseline bool) (map[string]map[string]int, error) { +func (e *Executor) callCoverageEndpoint(baseline bool) (CoverageSnapshot, error) { if !e.coverageEnabled || e.coveragePort == 0 { return nil, nil } @@ -68,8 +67,8 @@ func (e *Executor) callCoverageEndpoint(baseline bool) (map[string]map[string]in } var result struct { - OK bool `json:"ok"` - Coverage map[string]map[string]int `json:"coverage"` + OK bool `json:"ok"` + Coverage map[string]SnapshotFileCoverage `json:"coverage"` } if err := json.Unmarshal(body, &result); err != nil { return nil, fmt.Errorf("failed to parse coverage response: %w", err) @@ -79,30 +78,68 @@ func (e *Executor) callCoverageEndpoint(baseline bool) (map[string]map[string]in return nil, fmt.Errorf("coverage snapshot returned ok=false") } - // Normalize absolute paths to repo-relative paths - return normalizeFilePaths(result.Coverage), nil + // Convert to our internal format and normalize paths + snapshot := make(CoverageSnapshot) + for filePath, fileData := range result.Coverage { + snapshot[filePath] = FileCoverageData{ + Lines: fileData.Lines, + TotalBranches: fileData.TotalBranches, + CoveredBranches: fileData.CoveredBranches, + Branches: fileData.Branches, + } + } + + return normalizeCoveragePaths(snapshot), nil +} + +// SnapshotFileCoverage matches the JSON response from the SDK's /snapshot endpoint. +type SnapshotFileCoverage struct { + Lines map[string]int `json:"lines"` + TotalBranches int `json:"totalBranches"` + CoveredBranches int `json:"coveredBranches"` + Branches map[string]BranchInfo `json:"branches"` +} + +// BranchInfo tracks branch coverage at a specific line. +type BranchInfo struct { + Total int `json:"total"` + Covered int `json:"covered"` +} + +// FileCoverageData is the internal representation of per-file coverage. +type FileCoverageData struct { + Lines map[string]int + TotalBranches int + CoveredBranches int + Branches map[string]BranchInfo } +// CoverageSnapshot is the full coverage data for a snapshot. +type CoverageSnapshot map[string]FileCoverageData + // CoverageTestRecord holds per-test coverage data. type CoverageTestRecord struct { - TestID string - TestName string - LineCounts map[string]map[string]int // filePath -> lineNumber -> hitCount (per-test, not cumulative) + TestID string + TestName string + Coverage CoverageSnapshot } // CoverageFileDiff represents per-test coverage for a single file. type CoverageFileDiff struct { - CoveredLines []int `json:"covered_lines"` - CoverableLines int `json:"coverable_lines"` - CoveredCount int `json:"covered_count"` + CoveredLines []int `json:"covered_lines"` + CoverableLines int `json:"coverable_lines"` + CoveredCount int `json:"covered_count"` + TotalBranches int `json:"total_branches"` + CoveredBranches int `json:"covered_branches"` + Branches map[string]BranchInfo `json:"branches,omitempty"` } -// LinecountsToCoverageDetail converts raw line counts to CoverageFileDiff format. -func LinecountsToCoverageDetail(lineCounts map[string]map[string]int) map[string]CoverageFileDiff { +// SnapshotToCoverageDetail converts a CoverageSnapshot to per-file CoverageFileDiff format. +func SnapshotToCoverageDetail(snapshot CoverageSnapshot) map[string]CoverageFileDiff { result := make(map[string]CoverageFileDiff) - for filePath, lines := range lineCounts { + for filePath, fileData := range snapshot { var covered []int - for lineStr, count := range lines { + for lineStr, count := range fileData.Lines { if count > 0 { line, err := strconv.Atoi(lineStr) if err != nil || line <= 0 { @@ -115,9 +152,12 @@ func LinecountsToCoverageDetail(lineCounts map[string]map[string]int) map[string sort.Ints(covered) covered = dedup(covered) result[filePath] = CoverageFileDiff{ - CoveredLines: covered, - CoverableLines: len(lines), - CoveredCount: len(covered), + CoveredLines: covered, + CoverableLines: len(fileData.Lines), + CoveredCount: len(covered), + TotalBranches: fileData.TotalBranches, + CoveredBranches: fileData.CoveredBranches, + Branches: fileData.Branches, } } } @@ -137,35 +177,69 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { // then merge in per-test coverage. This gives accurate denominator. aggregate := mergeWithBaseline(e.coverageBaseline, records) - // Print summary to console return e.printCoverageSummary(records, aggregate) } // mergeWithBaseline creates aggregate coverage starting from the baseline // (which has ALL coverable lines including count=0), then merging in per-test data. -// If no baseline is available, falls back to merging per-test data only. -func mergeWithBaseline(baseline map[string]map[string]int, records []CoverageTestRecord) map[string]map[string]int { - merged := make(map[string]map[string]int) +func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) CoverageSnapshot { + merged := make(CoverageSnapshot) // Start with baseline (all coverable lines, count=0 for uncovered) if baseline != nil { - for filePath, lines := range baseline { - merged[filePath] = make(map[string]int) - for line, count := range lines { - merged[filePath][line] = count + for filePath, fileData := range baseline { + lines := make(map[string]int, len(fileData.Lines)) + for line, count := range fileData.Lines { + lines[line] = count + } + merged[filePath] = FileCoverageData{ + Lines: lines, + TotalBranches: fileData.TotalBranches, + CoveredBranches: fileData.CoveredBranches, + Branches: fileData.Branches, } } } - // Merge in per-test coverage (add counts) + // Merge in per-test coverage (add line counts, union branches) for _, record := range records { - for filePath, lines := range record.LineCounts { - if merged[filePath] == nil { - merged[filePath] = make(map[string]int) + for filePath, fileData := range record.Coverage { + existing, ok := merged[filePath] + if !ok { + existing = FileCoverageData{ + Lines: make(map[string]int), + Branches: make(map[string]BranchInfo), + } } - for line, count := range lines { - merged[filePath][line] += count + for line, count := range fileData.Lines { + existing.Lines[line] += count + } + // Merge branch data: take max of covered + for line, branchInfo := range fileData.Branches { + if eb, ok := existing.Branches[line]; ok { + if branchInfo.Total > eb.Total { + eb.Total = branchInfo.Total + } + if branchInfo.Covered > eb.Covered { + eb.Covered = branchInfo.Covered + } + existing.Branches[line] = eb + } else { + if existing.Branches == nil { + existing.Branches = make(map[string]BranchInfo) + } + existing.Branches[line] = branchInfo + } } + // Recompute branch totals from merged per-line data + totalB, covB := 0, 0 + for _, b := range existing.Branches { + totalB += b.Total + covB += b.Covered + } + existing.TotalBranches = totalB + existing.CoveredBranches = covB + merged[filePath] = existing } } @@ -182,17 +256,22 @@ type CoverageSummary struct { } type CoverageAggregate struct { - TotalCoverableLines int `json:"total_coverable_lines"` - TotalCoveredLines int `json:"total_covered_lines"` - CoveragePct float64 `json:"coverage_pct"` - TotalFiles int `json:"total_files"` - CoveredFiles int `json:"covered_files"` + TotalCoverableLines int `json:"total_coverable_lines"` + TotalCoveredLines int `json:"total_covered_lines"` + CoveragePct float64 `json:"coverage_pct"` + TotalFiles int `json:"total_files"` + CoveredFiles int `json:"covered_files"` + TotalBranches int `json:"total_branches"` + CoveredBranches int `json:"covered_branches"` + BranchCoveragePct float64 `json:"branch_coverage_pct"` } type CoverageFileSummary struct { - CoveredLines int `json:"covered_lines"` - CoverableLines int `json:"coverable_lines"` - CoveragePct float64 `json:"coverage_pct"` + CoveredLines int `json:"covered_lines"` + CoverableLines int `json:"coverable_lines"` + CoveragePct float64 `json:"coverage_pct"` + TotalBranches int `json:"total_branches"` + CoveredBranches int `json:"covered_branches"` } type CoverageTestSummary struct { @@ -202,7 +281,7 @@ type CoverageTestSummary struct { FilesTouched int `json:"files_touched"` } -func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate map[string]map[string]int) error { +func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) error { summary := CoverageSummary{ Timestamp: time.Now().Format(time.RFC3339), PerFile: make(map[string]CoverageFileSummary), @@ -210,18 +289,22 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate totalCoverable := 0 totalCovered := 0 + totalBranches := 0 + totalCoveredBranches := 0 coveredFiles := 0 - for filePath, lines := range aggregate { - coverable := len(lines) + for filePath, fileData := range aggregate { + coverable := len(fileData.Lines) covered := 0 - for _, count := range lines { + for _, count := range fileData.Lines { if count > 0 { covered++ } } totalCoverable += coverable totalCovered += covered + totalBranches += fileData.TotalBranches + totalCoveredBranches += fileData.CoveredBranches if covered > 0 { coveredFiles++ } @@ -230,7 +313,11 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate pct = float64(covered) / float64(coverable) * 100 } summary.PerFile[filePath] = CoverageFileSummary{ - CoveredLines: covered, CoverableLines: coverable, CoveragePct: pct, + CoveredLines: covered, + CoverableLines: coverable, + CoveragePct: pct, + TotalBranches: fileData.TotalBranches, + CoveredBranches: fileData.CoveredBranches, } } @@ -239,12 +326,20 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate aggPct = float64(totalCovered) / float64(totalCoverable) * 100 } + branchPct := 0.0 + if totalBranches > 0 { + branchPct = float64(totalCoveredBranches) / float64(totalBranches) * 100 + } + summary.Aggregate = CoverageAggregate{ TotalCoverableLines: totalCoverable, TotalCoveredLines: totalCovered, CoveragePct: aggPct, TotalFiles: len(aggregate), CoveredFiles: coveredFiles, + TotalBranches: totalBranches, + CoveredBranches: totalCoveredBranches, + BranchCoveragePct: branchPct, } // Per-test summaries from stored detail @@ -260,8 +355,12 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate } // Console output - log.Stderrln(fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% (%d/%d coverable lines across %d files)", - aggPct, totalCovered, totalCoverable, len(aggregate))) + coverageMsg := fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% lines (%d/%d)", aggPct, totalCovered, totalCoverable) + if totalBranches > 0 { + coverageMsg += fmt.Sprintf(", %.1f%% branches (%d/%d)", branchPct, totalCoveredBranches, totalBranches) + } + coverageMsg += fmt.Sprintf(" across %d files", len(aggregate)) + log.Stderrln(coverageMsg) type fileStat struct { path string @@ -319,24 +418,23 @@ func sanitizeFileName(name string) string { return replacer.Replace(name) } -// normalizeFilePaths converts absolute file paths to repo-relative paths. +// normalizeCoveragePaths converts absolute file paths to repo-relative paths. // Uses git root as the base (consistent across machines, handles monorepo // files outside the service directory like ../shared/utils.js). // Falls back to cwd if not in a git repo. -func normalizeFilePaths(lineCounts map[string]map[string]int) map[string]map[string]int { +func normalizeCoveragePaths(snapshot CoverageSnapshot) CoverageSnapshot { base := getPathNormalizationBase() if base == "" { - return lineCounts + return snapshot } - normalized := make(map[string]map[string]int, len(lineCounts)) - for absPath, lines := range lineCounts { + normalized := make(CoverageSnapshot, len(snapshot)) + for absPath, fileData := range snapshot { relPath, err := filepath.Rel(base, absPath) if err != nil || strings.HasPrefix(relPath, "..") { - // Outside the base - keep as-is rather than producing ../../... paths relPath = absPath } - normalized[relPath] = lines + normalized[relPath] = fileData } return normalized } diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go index 1e0622d..4ae2e01 100644 --- a/internal/runner/coverage_test.go +++ b/internal/runner/coverage_test.go @@ -53,85 +53,61 @@ func TestDedup(t *testing.T) { } } -func TestLinecountsToCoverageDetail(t *testing.T) { - t.Run("empty input", func(t *testing.T) { - result := LinecountsToCoverageDetail(nil) - assert.Empty(t, result) - }) +// Helper to create a simple FileCoverageData with just line counts +func makeFileData(lines map[string]int) FileCoverageData { + return FileCoverageData{Lines: lines} +} - t.Run("empty map", func(t *testing.T) { - result := LinecountsToCoverageDetail(map[string]map[string]int{}) +// Helper to create FileCoverageData with branches +func makeFileDataWithBranches(lines map[string]int, totalB, covB int, branches map[string]BranchInfo) FileCoverageData { + return FileCoverageData{ + Lines: lines, + TotalBranches: totalB, + CoveredBranches: covB, + Branches: branches, + } +} + +func TestSnapshotToCoverageDetail(t *testing.T) { + t.Run("empty input", func(t *testing.T) { + result := SnapshotToCoverageDetail(nil) assert.Empty(t, result) }) t.Run("single file with covered lines", func(t *testing.T) { - input := map[string]map[string]int{ - "/app/main.go": { - "1": 1, - "2": 3, - "5": 0, - "10": 1, - }, + input := CoverageSnapshot{ + "/app/main.go": makeFileData(map[string]int{"1": 1, "2": 3, "5": 0, "10": 1}), } - result := LinecountsToCoverageDetail(input) + result := SnapshotToCoverageDetail(input) require.Contains(t, result, "/app/main.go") - fd := result["/app/main.go"] assert.Equal(t, []int{1, 2, 10}, fd.CoveredLines) assert.Equal(t, 3, fd.CoveredCount) - assert.Equal(t, 4, fd.CoverableLines) // total lines in the map + assert.Equal(t, 4, fd.CoverableLines) }) t.Run("file with only zero counts is excluded", func(t *testing.T) { - input := map[string]map[string]int{ - "/app/unused.go": { - "1": 0, - "2": 0, - }, + input := CoverageSnapshot{ + "/app/unused.go": makeFileData(map[string]int{"1": 0, "2": 0}), } - result := LinecountsToCoverageDetail(input) + result := SnapshotToCoverageDetail(input) assert.Empty(t, result) }) - t.Run("invalid line number strings are skipped", func(t *testing.T) { - input := map[string]map[string]int{ - "/app/main.go": { - "abc": 1, - "0": 1, // line 0 is invalid (1-based) - "-1": 1, // negative line is invalid - "5": 1, // valid - }, + t.Run("includes branch data", func(t *testing.T) { + input := CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 1, "5": 1}, + 4, 2, + map[string]BranchInfo{"5": {Total: 2, Covered: 1}}, + ), } - result := LinecountsToCoverageDetail(input) - require.Contains(t, result, "/app/main.go") + result := SnapshotToCoverageDetail(input) fd := result["/app/main.go"] - assert.Equal(t, []int{5}, fd.CoveredLines) - assert.Equal(t, 1, fd.CoveredCount) - }) - - t.Run("multiple files", func(t *testing.T) { - input := map[string]map[string]int{ - "/app/a.go": {"1": 1, "2": 1}, - "/app/b.go": {"10": 2, "20": 0}, - } - result := LinecountsToCoverageDetail(input) - assert.Len(t, result, 2) - assert.Equal(t, 2, result["/app/a.go"].CoveredCount) - assert.Equal(t, 1, result["/app/b.go"].CoveredCount) - }) - - t.Run("covered lines are sorted and deduped", func(t *testing.T) { - input := map[string]map[string]int{ - "/app/main.go": { - "10": 1, - "3": 1, - "7": 1, - "1": 1, - }, - } - result := LinecountsToCoverageDetail(input) - require.Contains(t, result, "/app/main.go") - assert.Equal(t, []int{1, 3, 7, 10}, result["/app/main.go"].CoveredLines) + assert.Equal(t, 4, fd.TotalBranches) + assert.Equal(t, 2, fd.CoveredBranches) + assert.Equal(t, 2, fd.Branches["5"].Total) + assert.Equal(t, 1, fd.Branches["5"].Covered) }) } @@ -144,86 +120,85 @@ func TestMergeWithBaseline(t *testing.T) { t.Run("nil baseline with records", func(t *testing.T) { records := []CoverageTestRecord{ { - TestID: "test-1", - LineCounts: map[string]map[string]int{ - "/app/main.go": {"1": 1, "2": 3}, - }, + TestID: "test-1", + Coverage: CoverageSnapshot{"/app/main.go": makeFileData(map[string]int{"1": 1, "2": 3})}, }, } result := mergeWithBaseline(nil, records) require.Contains(t, result, "/app/main.go") - assert.Equal(t, 1, result["/app/main.go"]["1"]) - assert.Equal(t, 3, result["/app/main.go"]["2"]) + assert.Equal(t, 1, result["/app/main.go"].Lines["1"]) + assert.Equal(t, 3, result["/app/main.go"].Lines["2"]) }) t.Run("baseline with no records", func(t *testing.T) { - baseline := map[string]map[string]int{ - "/app/main.go": {"1": 0, "2": 0, "3": 0}, + baseline := CoverageSnapshot{ + "/app/main.go": makeFileData(map[string]int{"1": 0, "2": 0, "3": 0}), } result := mergeWithBaseline(baseline, nil) require.Contains(t, result, "/app/main.go") - assert.Equal(t, 0, result["/app/main.go"]["1"]) - assert.Equal(t, 0, result["/app/main.go"]["2"]) - assert.Equal(t, 0, result["/app/main.go"]["3"]) + assert.Equal(t, 0, result["/app/main.go"].Lines["1"]) + assert.Equal(t, 0, result["/app/main.go"].Lines["3"]) }) t.Run("baseline merged with records adds counts", func(t *testing.T) { - baseline := map[string]map[string]int{ - "/app/main.go": {"1": 0, "2": 0, "3": 0, "4": 0}, + baseline := CoverageSnapshot{ + "/app/main.go": makeFileData(map[string]int{"1": 0, "2": 0, "3": 0, "4": 0}), } records := []CoverageTestRecord{ - { - TestID: "test-1", - LineCounts: map[string]map[string]int{ - "/app/main.go": {"1": 1, "3": 2}, - }, - }, - { - TestID: "test-2", - LineCounts: map[string]map[string]int{ - "/app/main.go": {"1": 1, "4": 1}, - }, - }, + {TestID: "test-1", Coverage: CoverageSnapshot{"/app/main.go": makeFileData(map[string]int{"1": 1, "3": 2})}}, + {TestID: "test-2", Coverage: CoverageSnapshot{"/app/main.go": makeFileData(map[string]int{"1": 1, "4": 1})}}, } result := mergeWithBaseline(baseline, records) require.Contains(t, result, "/app/main.go") - assert.Equal(t, 2, result["/app/main.go"]["1"]) // 0 + 1 + 1 - assert.Equal(t, 0, result["/app/main.go"]["2"]) // baseline 0, no test coverage - assert.Equal(t, 2, result["/app/main.go"]["3"]) // 0 + 2 - assert.Equal(t, 1, result["/app/main.go"]["4"]) // 0 + 1 + assert.Equal(t, 2, result["/app/main.go"].Lines["1"]) // 0+1+1 + assert.Equal(t, 0, result["/app/main.go"].Lines["2"]) // baseline 0, no test + assert.Equal(t, 2, result["/app/main.go"].Lines["3"]) // 0+2 + assert.Equal(t, 1, result["/app/main.go"].Lines["4"]) // 0+1 }) t.Run("records can add new files not in baseline", func(t *testing.T) { - baseline := map[string]map[string]int{ - "/app/main.go": {"1": 0}, + baseline := CoverageSnapshot{ + "/app/main.go": makeFileData(map[string]int{"1": 0}), } records := []CoverageTestRecord{ - { - TestID: "test-1", - LineCounts: map[string]map[string]int{ - "/app/new_file.go": {"10": 5}, - }, - }, + {TestID: "test-1", Coverage: CoverageSnapshot{"/app/new.go": makeFileData(map[string]int{"10": 5})}}, } result := mergeWithBaseline(baseline, records) assert.Len(t, result, 2) - assert.Equal(t, 5, result["/app/new_file.go"]["10"]) + assert.Equal(t, 5, result["/app/new.go"].Lines["10"]) }) t.Run("baseline is not mutated", func(t *testing.T) { - baseline := map[string]map[string]int{ - "/app/main.go": {"1": 0}, + baseline := CoverageSnapshot{ + "/app/main.go": makeFileData(map[string]int{"1": 0}), } records := []CoverageTestRecord{ - { - TestID: "test-1", - LineCounts: map[string]map[string]int{ - "/app/main.go": {"1": 5}, - }, - }, + {TestID: "test-1", Coverage: CoverageSnapshot{"/app/main.go": makeFileData(map[string]int{"1": 5})}}, } _ = mergeWithBaseline(baseline, records) - // Original baseline should be untouched - assert.Equal(t, 0, baseline["/app/main.go"]["1"]) + assert.Equal(t, 0, baseline["/app/main.go"].Lines["1"]) + }) + + t.Run("merges branch data", func(t *testing.T) { + baseline := CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 0}, + 4, 0, + map[string]BranchInfo{"5": {Total: 2, Covered: 0}}, + ), + } + records := []CoverageTestRecord{ + {TestID: "test-1", Coverage: CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 1}, + 2, 1, + map[string]BranchInfo{"5": {Total: 2, Covered: 1}}, + ), + }}, + } + result := mergeWithBaseline(baseline, records) + // Branch data should be merged (max of covered) + assert.Equal(t, 1, result["/app/main.go"].Branches["5"].Covered) + assert.Equal(t, 2, result["/app/main.go"].Branches["5"].Total) }) } diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 3d3387c..37e46d1 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -98,7 +98,7 @@ type Executor struct { coveragePort int // Coverage snapshot server port coveragePerTest map[string]map[string]CoverageFileDiff coveragePerTestMu sync.Mutex - coverageBaseline map[string]map[string]int + coverageBaseline CoverageSnapshot } func NewExecutor() *Executor { @@ -492,27 +492,38 @@ func (e *Executor) IsCoverageEnabled() bool { // SetCoverageBaseline merges new baseline data into the existing baseline. // Called per environment group - accumulates across service restarts. -func (e *Executor) SetCoverageBaseline(baseline map[string]map[string]int) { +func (e *Executor) SetCoverageBaseline(baseline CoverageSnapshot) { if e.coverageBaseline == nil { - e.coverageBaseline = make(map[string]map[string]int) - } - for filePath, lines := range baseline { - if e.coverageBaseline[filePath] == nil { - e.coverageBaseline[filePath] = make(map[string]int) + e.coverageBaseline = make(CoverageSnapshot) + } + for filePath, fileData := range baseline { + existing, ok := e.coverageBaseline[filePath] + if !ok { + existing = FileCoverageData{ + Lines: make(map[string]int), + Branches: make(map[string]BranchInfo), + } } - for line, count := range lines { - // Keep the existing count if it's already tracked (don't overwrite covered with uncovered) - if existing, ok := e.coverageBaseline[filePath][line]; !ok || existing == 0 { - e.coverageBaseline[filePath][line] = count + for line, count := range fileData.Lines { + if existingCount, ok := existing.Lines[line]; !ok || existingCount == 0 { + existing.Lines[line] = count } } + // Merge branch data (keep max) + for line, branchInfo := range fileData.Branches { + if existing.Branches == nil { + existing.Branches = make(map[string]BranchInfo) + } + if eb, ok := existing.Branches[line]; !ok || branchInfo.Total > eb.Total { + existing.Branches[line] = branchInfo + } + } + existing.TotalBranches = fileData.TotalBranches + existing.CoveredBranches = fileData.CoveredBranches + e.coverageBaseline[filePath] = existing } } -func (e *Executor) GetCoverageBaseline() map[string]map[string]int { - return e.coverageBaseline -} - // SetTestCoverageDetail stores per-test coverage diff for display in TUI/print. func (e *Executor) SetTestCoverageDetail(testID string, detail map[string]CoverageFileDiff) { e.coveragePerTestMu.Lock() From f19704783c95c81ca97e0b0167c71044c34f007e Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 18:42:26 -0700 Subject: [PATCH 12/36] feat: set TS_NODE_EMIT=true for ts-node coverage support --- internal/runner/service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/runner/service.go b/internal/runner/service.go index 46ae227..bb3a134 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -164,6 +164,9 @@ func (e *Executor) StartService() error { } env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", v8CoverageDir)) env = append(env, fmt.Sprintf("TUSK_COVERAGE_PORT=%d", e.coveragePort)) + // ts-node: force emit compiled JS + source maps to disk so we can read them. + // Without this, ts-node compiles in memory and our SDK can't find the compiled output. + env = append(env, "TS_NODE_EMIT=true") log.Debug("Coverage enabled", "v8_dir", v8CoverageDir, "port", e.coveragePort) } From 99bf7bb91497fab3ead2474465f6b5f7061d284c Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 19:19:28 -0700 Subject: [PATCH 13/36] feat: migrate coverage from HTTP to protobuf channel --- go.mod | 2 ++ internal/runner/coverage.go | 68 +++++++++++-------------------------- internal/runner/executor.go | 1 - internal/runner/server.go | 60 ++++++++++++++++++++++++++++++++ internal/runner/service.go | 9 ++--- 5 files changed, 84 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index 8f2f66b..a3f9502 100644 --- a/go.mod +++ b/go.mod @@ -74,3 +74,5 @@ require ( golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.29.0 // indirect ) + +replace github.com/Use-Tusk/tusk-drift-schemas => /Users/sohil/projects/tusk-drift-schemas diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 57f6271..f58a344 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -1,10 +1,7 @@ package runner import ( - "encoding/json" "fmt" - "io" - "net/http" "os" "path/filepath" "sort" @@ -16,8 +13,6 @@ import ( "github.com/Use-Tusk/tusk-cli/internal/utils" ) -const coverageSnapshotTimeout = 5 * time.Second - // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. // Returns per-file coverage data for this test only (counters auto-reset). func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { @@ -41,65 +36,42 @@ func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { } func (e *Executor) callCoverageEndpoint(baseline bool) (CoverageSnapshot, error) { - if !e.coverageEnabled || e.coveragePort == 0 { + if !e.coverageEnabled || e.server == nil { return nil, nil } - url := fmt.Sprintf("http://127.0.0.1:%d/snapshot", e.coveragePort) - if baseline { - url += "?baseline=true" - } - httpClient := &http.Client{Timeout: coverageSnapshotTimeout} - - resp, err := httpClient.Get(url) + resp, err := e.server.SendCoverageSnapshot(baseline) if err != nil { return nil, fmt.Errorf("coverage snapshot failed: %w", err) } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read coverage response: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("coverage snapshot status %d: %s", resp.StatusCode, string(body)) - } - var result struct { - OK bool `json:"ok"` - Coverage map[string]SnapshotFileCoverage `json:"coverage"` - } - if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to parse coverage response: %w", err) - } + // Convert protobuf response to our internal format + snapshot := make(CoverageSnapshot) + for filePath, fileData := range resp.Coverage { + branches := make(map[string]BranchInfo) + for line, branchProto := range fileData.Branches { + branches[line] = BranchInfo{ + Total: int(branchProto.Total), + Covered: int(branchProto.Covered), + } + } - if !result.OK { - return nil, fmt.Errorf("coverage snapshot returned ok=false") - } + lines := make(map[string]int) + for line, count := range fileData.Lines { + lines[line] = int(count) + } - // Convert to our internal format and normalize paths - snapshot := make(CoverageSnapshot) - for filePath, fileData := range result.Coverage { snapshot[filePath] = FileCoverageData{ - Lines: fileData.Lines, - TotalBranches: fileData.TotalBranches, - CoveredBranches: fileData.CoveredBranches, - Branches: fileData.Branches, + Lines: lines, + TotalBranches: int(fileData.TotalBranches), + CoveredBranches: int(fileData.CoveredBranches), + Branches: branches, } } return normalizeCoveragePaths(snapshot), nil } -// SnapshotFileCoverage matches the JSON response from the SDK's /snapshot endpoint. -type SnapshotFileCoverage struct { - Lines map[string]int `json:"lines"` - TotalBranches int `json:"totalBranches"` - CoveredBranches int `json:"coveredBranches"` - Branches map[string]BranchInfo `json:"branches"` -} - // BranchInfo tracks branch coverage at a specific line. type BranchInfo struct { Total int `json:"total"` diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 37e46d1..6db5fe5 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -95,7 +95,6 @@ type Executor struct { // Coverage coverageEnabled bool - coveragePort int // Coverage snapshot server port coveragePerTest map[string]map[string]CoverageFileDiff coveragePerTestMu sync.Mutex coverageBaseline CoverageSnapshot diff --git a/internal/runner/server.go b/internal/runner/server.go index c9f0b4f..e3d1d3e 100644 --- a/internal/runner/server.go +++ b/internal/runner/server.go @@ -696,6 +696,9 @@ func (ms *Server) handleConnection(conn net.Conn) { case core.MessageType_MESSAGE_TYPE_SET_TIME_TRAVEL: // SDK is responding to our SetTimeTravel request ms.handleSetTimeTravelResponse(&sdkMsg) + case core.MessageType_MESSAGE_TYPE_COVERAGE_SNAPSHOT: + // SDK is responding to our CoverageSnapshot request + ms.handleCoverageSnapshotResponse(&sdkMsg) default: log.Debug("Unknown message type", "type", sdkMsg.Type) } @@ -982,6 +985,63 @@ func (ms *Server) waitForSDKResponse(requestID string, timeout time.Duration) (* } } +// SendCoverageSnapshot sends a coverage snapshot request to the SDK and waits for the response. +// Returns per-file coverage data. If baseline=true, includes all coverable lines (count=0 for uncovered). +func (ms *Server) SendCoverageSnapshot(baseline bool) (*core.CoverageSnapshotResponse, error) { + ms.mu.RLock() + conn := ms.sdkConnection + ms.mu.RUnlock() + + if conn == nil { + return nil, fmt.Errorf("no SDK connection available") + } + + requestID := fmt.Sprintf("coverage-%d", time.Now().UnixNano()) + + msg := &core.CLIMessage{ + Type: core.MessageType_MESSAGE_TYPE_COVERAGE_SNAPSHOT, + RequestId: requestID, + Payload: &core.CLIMessage_CoverageSnapshotRequest{ + CoverageSnapshotRequest: &core.CoverageSnapshotRequest{ + Baseline: baseline, + }, + }, + } + + if err := ms.sendProtobufResponse(conn, msg); err != nil { + return nil, fmt.Errorf("failed to send coverage snapshot request: %w", err) + } + + response, err := ms.waitForSDKResponse(requestID, 10*time.Second) + if err != nil { + return nil, fmt.Errorf("failed to receive coverage snapshot response: %w", err) + } + + coverageResp := response.GetCoverageSnapshotResponse() + if coverageResp == nil { + return nil, fmt.Errorf("unexpected response type for coverage snapshot") + } + + if !coverageResp.Success { + return nil, fmt.Errorf("SDK coverage snapshot failed: %s", coverageResp.Error) + } + + return coverageResp, nil +} + +// handleCoverageSnapshotResponse routes coverage snapshot responses to pending request channels +func (ms *Server) handleCoverageSnapshotResponse(msg *core.SDKMessage) { + ms.pendingMu.Lock() + respChan, ok := ms.pendingRequests[msg.RequestId] + ms.pendingMu.Unlock() + + if ok { + respChan <- msg + } else { + log.Debug("Received coverage snapshot response with unknown request ID", "requestId", msg.RequestId) + } +} + // handleSetTimeTravelResponse routes SetTimeTravel responses to pending request channels func (ms *Server) handleSetTimeTravelResponse(msg *core.SDKMessage) { ms.pendingMu.Lock() diff --git a/internal/runner/service.go b/internal/runner/service.go index bb3a134..c54d3be 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -45,10 +45,7 @@ func (e *Executor) StartService() error { command := cfg.Service.Start.Command - // Coverage: set port (env vars injected after sandbox wrapping) - if e.coverageEnabled && e.coveragePort == 0 { - e.coveragePort = 19876 - } + // Coverage: nothing to set here, env vars injected below after sandbox wrapping // Wrap command with fence sandboxing (if supported and enabled) replayOverridePath := e.getReplayComposeOverride() @@ -163,11 +160,9 @@ func (e *Executor) StartService() error { return fmt.Errorf("failed to create temp dir for V8 coverage: %w", err) } env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", v8CoverageDir)) - env = append(env, fmt.Sprintf("TUSK_COVERAGE_PORT=%d", e.coveragePort)) // ts-node: force emit compiled JS + source maps to disk so we can read them. - // Without this, ts-node compiles in memory and our SDK can't find the compiled output. env = append(env, "TS_NODE_EMIT=true") - log.Debug("Coverage enabled", "v8_dir", v8CoverageDir, "port", e.coveragePort) + log.Debug("Coverage enabled", "v8_dir", v8CoverageDir) } e.serviceCmd.Env = env From ff88435f23d1a1c303d5caebb50edc920c5ca90a Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 19:30:30 -0700 Subject: [PATCH 14/36] fix: remove debug logging from server.go --- internal/runner/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runner/server.go b/internal/runner/server.go index e3d1d3e..b17dc6e 100644 --- a/internal/runner/server.go +++ b/internal/runner/server.go @@ -697,7 +697,7 @@ func (ms *Server) handleConnection(conn net.Conn) { // SDK is responding to our SetTimeTravel request ms.handleSetTimeTravelResponse(&sdkMsg) case core.MessageType_MESSAGE_TYPE_COVERAGE_SNAPSHOT: - // SDK is responding to our CoverageSnapshot request + log.Debug("Received coverage snapshot response", "requestId", sdkMsg.RequestId) ms.handleCoverageSnapshotResponse(&sdkMsg) default: log.Debug("Unknown message type", "type", sdkMsg.Type) From 67eea36f691f2ff33561a96afcd625ac46071696 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 19:35:49 -0700 Subject: [PATCH 15/36] fix: critical bugs from code review - Fix shallow copy bug in mergeWithBaseline: baseline branches map was shared, causing mutation of baseline data during merge - Fix branch merge to use UNION semantics (sum covered, clamp to total) instead of max. Test A covers path 1, test B covers path 2 = both covered. - Fix shallow copy in SnapshotToCoverageDetail branches - Extract retry constants (coverageBaselineMaxRetries, coverageBaselineRetryDelay) - Remove dead code: sanitizeFileName() and its tests (from file-output era) --- internal/runner/coverage.go | 68 +++++++++++++++++++------------- internal/runner/coverage_test.go | 24 ----------- 2 files changed, 41 insertions(+), 51 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index f58a344..f332b02 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -13,6 +13,12 @@ import ( "github.com/Use-Tusk/tusk-cli/internal/utils" ) +const ( + coverageBaselineMaxRetries = 15 + coverageBaselineRetryDelay = 200 * time.Millisecond + coverageSnapshotTimeout = 10 * time.Second +) + // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. // Returns per-file coverage data for this test only (counters auto-reset). func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { @@ -24,13 +30,13 @@ func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { // Retries with backoff since the coverage server may not be ready immediately after service start. func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { var lastErr error - for attempt := 0; attempt < 15; attempt++ { + for attempt := 0; attempt < coverageBaselineMaxRetries; attempt++ { result, err := e.callCoverageEndpoint(true) if err == nil { return result, nil } lastErr = err - time.Sleep(200 * time.Millisecond) + time.Sleep(coverageBaselineRetryDelay) } return nil, fmt.Errorf("coverage baseline failed after retries: %w", lastErr) } @@ -123,13 +129,18 @@ func SnapshotToCoverageDetail(snapshot CoverageSnapshot) map[string]CoverageFile if len(covered) > 0 { sort.Ints(covered) covered = dedup(covered) + // Deep-copy branches to avoid shared references + branchesCopy := make(map[string]BranchInfo, len(fileData.Branches)) + for line, info := range fileData.Branches { + branchesCopy[line] = info + } result[filePath] = CoverageFileDiff{ CoveredLines: covered, CoverableLines: len(fileData.Lines), CoveredCount: len(covered), TotalBranches: fileData.TotalBranches, CoveredBranches: fileData.CoveredBranches, - Branches: fileData.Branches, + Branches: branchesCopy, } } } @@ -152,28 +163,36 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { return e.printCoverageSummary(records, aggregate) } -// mergeWithBaseline creates aggregate coverage starting from the baseline -// (which has ALL coverable lines including count=0), then merging in per-test data. +// mergeWithBaseline creates aggregate coverage by starting from the baseline +// (all coverable lines including count=0) and unioning per-test data. +// +// Branch merging uses UNION semantics: if test A covers branch path 1 and test B +// covers branch path 2, the aggregate shows both paths as covered. This is done +// by summing covered counts per line (clamped to total) rather than taking max. func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) CoverageSnapshot { merged := make(CoverageSnapshot) - // Start with baseline (all coverable lines, count=0 for uncovered) + // Deep-copy baseline (don't mutate the original) if baseline != nil { for filePath, fileData := range baseline { lines := make(map[string]int, len(fileData.Lines)) for line, count := range fileData.Lines { lines[line] = count } + branches := make(map[string]BranchInfo, len(fileData.Branches)) + for line, info := range fileData.Branches { + branches[line] = info // BranchInfo is a value type, safe to copy + } merged[filePath] = FileCoverageData{ Lines: lines, TotalBranches: fileData.TotalBranches, CoveredBranches: fileData.CoveredBranches, - Branches: fileData.Branches, + Branches: branches, } } } - // Merge in per-test coverage (add line counts, union branches) + // Union per-test coverage into the merged result for _, record := range records { for filePath, fileData := range record.Coverage { existing, ok := merged[filePath] @@ -183,27 +202,26 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) Branches: make(map[string]BranchInfo), } } + // Add line counts for line, count := range fileData.Lines { existing.Lines[line] += count } - // Merge branch data: take max of covered + // Union branch data: sum covered counts (clamped to total) for line, branchInfo := range fileData.Branches { - if eb, ok := existing.Branches[line]; ok { - if branchInfo.Total > eb.Total { - eb.Total = branchInfo.Total - } - if branchInfo.Covered > eb.Covered { - eb.Covered = branchInfo.Covered - } - existing.Branches[line] = eb - } else { - if existing.Branches == nil { - existing.Branches = make(map[string]BranchInfo) - } - existing.Branches[line] = branchInfo + if existing.Branches == nil { + existing.Branches = make(map[string]BranchInfo) } + eb := existing.Branches[line] + if branchInfo.Total > eb.Total { + eb.Total = branchInfo.Total + } + eb.Covered += branchInfo.Covered + if eb.Covered > eb.Total { + eb.Covered = eb.Total // Clamp: can't cover more branches than exist + } + existing.Branches[line] = eb } - // Recompute branch totals from merged per-line data + // Recompute file-level branch totals from per-line data totalB, covB := 0, 0 for _, b := range existing.Branches { totalB += b.Total @@ -385,10 +403,6 @@ func dedup(sorted []int) []int { return result } -func sanitizeFileName(name string) string { - replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_") - return replacer.Replace(name) -} // normalizeCoveragePaths converts absolute file paths to repo-relative paths. // Uses git root as the base (consistent across machines, handles monorepo diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go index 4ae2e01..5899cab 100644 --- a/internal/runner/coverage_test.go +++ b/internal/runner/coverage_test.go @@ -7,30 +7,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestSanitizeFileName(t *testing.T) { - tests := []struct { - name string - input string - expected string - }{ - {name: "no special chars", input: "simple", expected: "simple"}, - {name: "forward slashes", input: "a/b/c", expected: "a_b_c"}, - {name: "backslashes", input: "a\\b\\c", expected: "a_b_c"}, - {name: "colons", input: "a:b:c", expected: "a_b_c"}, - {name: "spaces", input: "a b c", expected: "a_b_c"}, - {name: "mixed separators", input: "path/to:some file\\here", expected: "path_to_some_file_here"}, - {name: "empty string", input: "", expected: ""}, - {name: "already clean", input: "test_id_123", expected: "test_id_123"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := sanitizeFileName(tt.input) - assert.Equal(t, tt.expected, result) - }) - } -} - func TestDedup(t *testing.T) { tests := []struct { name string From cc4f32c67143a706ed140b60dea2e1369e417f32 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 19:41:00 -0700 Subject: [PATCH 16/36] refactor: split printCoverageSummary, add tests, cleanup - Extract ComputeCoverageSummary() as pure testable function (no I/O) - printCoverageSummary() now just formats and prints - Add tests for ComputeCoverageSummary (empty, percentages, per-file, branches, per-test) - Add tests for branch union semantics and baseline immutability - Add tests for normalizeCoveragePaths edge cases - Remove unused os/filepath import from print function --- internal/runner/coverage.go | 45 ++++++----- internal/runner/coverage_test.go | 131 ++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 19 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index f332b02..d1c5d6a 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -271,7 +271,13 @@ type CoverageTestSummary struct { FilesTouched int `json:"files_touched"` } -func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) error { +// ComputeCoverageSummary builds a CoverageSummary from aggregate coverage data +// and per-test detail. This is a pure function (no side effects, no I/O). +func ComputeCoverageSummary( + aggregate CoverageSnapshot, + perTestDetail map[string]map[string]CoverageFileDiff, + records []CoverageTestRecord, +) CoverageSummary { summary := CoverageSummary{ Timestamp: time.Now().Format(time.RFC3339), PerFile: make(map[string]CoverageFileSummary), @@ -315,7 +321,6 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate if totalCoverable > 0 { aggPct = float64(totalCovered) / float64(totalCoverable) * 100 } - branchPct := 0.0 if totalBranches > 0 { branchPct = float64(totalCoveredBranches) / float64(totalBranches) * 100 @@ -332,10 +337,9 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate BranchCoveragePct: branchPct, } - // Per-test summaries from stored detail for _, record := range records { ts := CoverageTestSummary{TestID: record.TestID, TestName: record.TestName} - if detail := e.GetTestCoverageDetail(record.TestID); detail != nil { + if detail, ok := perTestDetail[record.TestID]; ok { for _, fd := range detail { ts.CoveredLines += fd.CoveredCount } @@ -344,17 +348,27 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate summary.PerTest = append(summary.PerTest, ts) } - // Console output - coverageMsg := fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% lines (%d/%d)", aggPct, totalCovered, totalCoverable) - if totalBranches > 0 { - coverageMsg += fmt.Sprintf(", %.1f%% branches (%d/%d)", branchPct, totalCoveredBranches, totalBranches) + return summary +} + +// printCoverageSummary computes and prints the coverage summary to stderr. +func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) error { + summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) + + // Aggregate line + coverageMsg := fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% lines (%d/%d)", + summary.Aggregate.CoveragePct, summary.Aggregate.TotalCoveredLines, summary.Aggregate.TotalCoverableLines) + if summary.Aggregate.TotalBranches > 0 { + coverageMsg += fmt.Sprintf(", %.1f%% branches (%d/%d)", + summary.Aggregate.BranchCoveragePct, summary.Aggregate.CoveredBranches, summary.Aggregate.TotalBranches) } - coverageMsg += fmt.Sprintf(" across %d files", len(aggregate)) + coverageMsg += fmt.Sprintf(" across %d files", summary.Aggregate.TotalFiles) log.Stderrln(coverageMsg) + // Per-file breakdown sorted by coverage % type fileStat struct { - path string - pct float64 + path string + pct float64 cov, tot int } var stats []fileStat @@ -367,15 +381,10 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate log.Stderrln("\n Per-file:") for _, s := range stats { - shortPath := s.path - if cwd, err := os.Getwd(); err == nil { - if rel, err := filepath.Rel(cwd, s.path); err == nil { - shortPath = rel - } - } - log.Stderrln(fmt.Sprintf(" %-40s %5.1f%% (%d/%d)", shortPath, s.pct, s.cov, s.tot)) + log.Stderrln(fmt.Sprintf(" %-40s %5.1f%% (%d/%d)", s.path, s.pct, s.cov, s.tot)) } + // Per-test breakdown log.Stderrln("\n Per-test:") for _, ts := range summary.PerTest { name := ts.TestName diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go index 5899cab..f1ab6ea 100644 --- a/internal/runner/coverage_test.go +++ b/internal/runner/coverage_test.go @@ -173,8 +173,137 @@ func TestMergeWithBaseline(t *testing.T) { }}, } result := mergeWithBaseline(baseline, records) - // Branch data should be merged (max of covered) assert.Equal(t, 1, result["/app/main.go"].Branches["5"].Covered) assert.Equal(t, 2, result["/app/main.go"].Branches["5"].Total) }) + + t.Run("branch union semantics: two tests cover different branches", func(t *testing.T) { + baseline := CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 0}, + 2, 0, + map[string]BranchInfo{"5": {Total: 2, Covered: 0}}, + ), + } + records := []CoverageTestRecord{ + {TestID: "test-1", Coverage: CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 1}, + 2, 1, + map[string]BranchInfo{"5": {Total: 2, Covered: 1}}, // test 1 covers 1 branch + ), + }}, + {TestID: "test-2", Coverage: CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 1}, + 2, 1, + map[string]BranchInfo{"5": {Total: 2, Covered: 1}}, // test 2 covers 1 branch + ), + }}, + } + result := mergeWithBaseline(baseline, records) + // Union: 1 + 1 = 2, clamped to total 2 + assert.Equal(t, 2, result["/app/main.go"].Branches["5"].Covered) + assert.Equal(t, 2, result["/app/main.go"].Branches["5"].Total) + }) + + t.Run("baseline branches not mutated", func(t *testing.T) { + baseline := CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 0}, + 2, 0, + map[string]BranchInfo{"5": {Total: 2, Covered: 0}}, + ), + } + records := []CoverageTestRecord{ + {TestID: "test-1", Coverage: CoverageSnapshot{ + "/app/main.go": makeFileDataWithBranches( + map[string]int{"1": 1}, + 2, 1, + map[string]BranchInfo{"5": {Total: 2, Covered: 1}}, + ), + }}, + } + _ = mergeWithBaseline(baseline, records) + // Original baseline branches should be untouched + assert.Equal(t, 0, baseline["/app/main.go"].Branches["5"].Covered) + }) +} + +func TestComputeCoverageSummary(t *testing.T) { + t.Run("empty aggregate", func(t *testing.T) { + summary := ComputeCoverageSummary(nil, nil, nil) + assert.Equal(t, 0, summary.Aggregate.TotalCoverableLines) + assert.Equal(t, 0.0, summary.Aggregate.CoveragePct) + }) + + t.Run("computes aggregate percentages", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "main.go": makeFileData(map[string]int{"1": 1, "2": 1, "3": 0, "4": 0}), + } + summary := ComputeCoverageSummary(aggregate, nil, nil) + assert.Equal(t, 4, summary.Aggregate.TotalCoverableLines) + assert.Equal(t, 2, summary.Aggregate.TotalCoveredLines) + assert.Equal(t, 50.0, summary.Aggregate.CoveragePct) + assert.Equal(t, 1, summary.Aggregate.TotalFiles) + assert.Equal(t, 1, summary.Aggregate.CoveredFiles) + }) + + t.Run("computes per-file summaries", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "a.go": makeFileData(map[string]int{"1": 1, "2": 0}), + "b.go": makeFileData(map[string]int{"1": 1, "2": 1}), + } + summary := ComputeCoverageSummary(aggregate, nil, nil) + assert.Equal(t, 50.0, summary.PerFile["a.go"].CoveragePct) + assert.Equal(t, 100.0, summary.PerFile["b.go"].CoveragePct) + }) + + t.Run("includes branch coverage", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "main.go": makeFileDataWithBranches( + map[string]int{"1": 1}, + 4, 2, + map[string]BranchInfo{"5": {Total: 2, Covered: 1}}, + ), + } + summary := ComputeCoverageSummary(aggregate, nil, nil) + assert.Equal(t, 4, summary.Aggregate.TotalBranches) + assert.Equal(t, 2, summary.Aggregate.CoveredBranches) + assert.Equal(t, 50.0, summary.Aggregate.BranchCoveragePct) + }) + + t.Run("includes per-test summaries", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "main.go": makeFileData(map[string]int{"1": 1}), + } + perTest := map[string]map[string]CoverageFileDiff{ + "test-1": {"main.go": {CoveredCount: 5, CoverableLines: 10}}, + "test-2": {"main.go": {CoveredCount: 3, CoverableLines: 10}}, + } + records := []CoverageTestRecord{ + {TestID: "test-1", TestName: "GET /api"}, + {TestID: "test-2", TestName: "POST /api"}, + } + summary := ComputeCoverageSummary(aggregate, perTest, records) + require.Len(t, summary.PerTest, 2) + assert.Equal(t, 5, summary.PerTest[0].CoveredLines) + assert.Equal(t, "GET /api", summary.PerTest[0].TestName) + assert.Equal(t, 3, summary.PerTest[1].CoveredLines) + }) +} + +func TestNormalizeCoveragePaths(t *testing.T) { + t.Run("nil input returns empty", func(t *testing.T) { + result := normalizeCoveragePaths(nil) + assert.Len(t, result, 0) + }) + + t.Run("empty input returns empty", func(t *testing.T) { + result := normalizeCoveragePaths(CoverageSnapshot{}) + assert.Empty(t, result) + }) + + // Note: full path normalization depends on git root which is environment-specific. + // We test the function handles edge cases; full integration is tested E2E. } From a7d41851dc0199bdf59d6e1ae72d21a2ac84a729 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 19:46:37 -0700 Subject: [PATCH 17/36] fix: prod readiness - overflow guard, parse error logging - Add overflow guard in branch coverage accumulation (clamp + negative check) - Log debug warning for invalid line numbers instead of silent skip --- internal/runner/coverage.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index d1c5d6a..ecbbf49 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -121,6 +121,7 @@ func SnapshotToCoverageDetail(snapshot CoverageSnapshot) map[string]CoverageFile if count > 0 { line, err := strconv.Atoi(lineStr) if err != nil || line <= 0 { + log.Debug("Skipping invalid line number in coverage data", "line", lineStr, "file", filePath) continue } covered = append(covered, line) @@ -215,9 +216,11 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) if branchInfo.Total > eb.Total { eb.Total = branchInfo.Total } - eb.Covered += branchInfo.Covered - if eb.Covered > eb.Total { - eb.Covered = eb.Total // Clamp: can't cover more branches than exist + newCovered := eb.Covered + branchInfo.Covered + if newCovered > eb.Total || newCovered < 0 { // Clamp + overflow guard + eb.Covered = eb.Total + } else { + eb.Covered = newCovered } existing.Branches[line] = eb } From 19cb788773876cc1d00a9d3d487ecb677222d1d6 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 19:54:44 -0700 Subject: [PATCH 18/36] fix: warn user when baseline fails and coverage denominator is incomplete --- internal/runner/coverage.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index ecbbf49..a359ae9 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -366,6 +366,9 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate summary.Aggregate.BranchCoveragePct, summary.Aggregate.CoveredBranches, summary.Aggregate.TotalBranches) } coverageMsg += fmt.Sprintf(" across %d files", summary.Aggregate.TotalFiles) + if e.coverageBaseline == nil { + coverageMsg += " ⚠️ baseline failed - denominator may be incomplete" + } log.Stderrln(coverageMsg) // Per-file breakdown sorted by coverage % From 61eeba8a669542838b95c005304ab781a2168405 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Wed, 1 Apr 2026 20:10:20 -0700 Subject: [PATCH 19/36] feat: add TUSK_COVERAGE env var as language-agnostic coverage signal --- internal/runner/service.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/runner/service.go b/internal/runner/service.go index c54d3be..c1b69fe 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -152,15 +152,18 @@ func (e *Executor) StartService() error { // Coverage: inject env vars that SDK coverage servers listen for. // NODE_V8_COVERAGE is required by the Node SDK to enable V8 coverage collection. - // TUSK_COVERAGE_PORT tells both Node and Python SDKs which port to serve snapshots on. + // Coverage env vars: + // TUSK_COVERAGE=true - language-agnostic signal for both Node and Python SDKs + // NODE_V8_COVERAGE= - Node-specific: tells V8 to collect coverage data + // TS_NODE_EMIT=true - Node-specific: forces ts-node to write compiled JS to disk if e.coverageEnabled { - // Use temp dir for V8 coverage files (SDK reads + deletes immediately, nothing persists) + env = append(env, "TUSK_COVERAGE=true") + // Node.js: V8 coverage needs a directory to write JSON files v8CoverageDir, err := os.MkdirTemp("", "tusk-v8-coverage-*") if err != nil { return fmt.Errorf("failed to create temp dir for V8 coverage: %w", err) } env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", v8CoverageDir)) - // ts-node: force emit compiled JS + source maps to disk so we can read them. env = append(env, "TS_NODE_EMIT=true") log.Debug("Coverage enabled", "v8_dir", v8CoverageDir) } From 117766d8623916499ac9ef227ade261d4c8187ae Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Fri, 3 Apr 2026 14:35:52 -0700 Subject: [PATCH 20/36] docs: add code coverage reference documentation Add docs/drift/coverage.md with CLI flags, config options, output formats, Docker Compose setup, and limitations. Update configuration.md with coverage section (enabled, include, exclude, strip_path_prefix). --- docs/drift/configuration.md | 41 +++++++ docs/drift/coverage.md | 228 ++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 docs/drift/coverage.md diff --git a/docs/drift/configuration.md b/docs/drift/configuration.md index fe2d930..e231317 100644 --- a/docs/drift/configuration.md +++ b/docs/drift/configuration.md @@ -392,6 +392,47 @@ This will not affect CLI behavior. See SDK for more details: +## Coverage + +Configuration for code coverage collection. See [`docs/drift/coverage.md`](coverage.md) for full documentation. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyTypeDefaultDescription
coverage.enabledboolfalseWhen true, automatically collect coverage during suite validation runs on the default branch. No CI changes needed.
coverage.includestring[](all files)Only include files matching at least one pattern. Supports ** for recursive matching. Paths are git-relative.
coverage.excludestring[](none)Exclude files matching any pattern. Applied after include. Supports ** for recursive matching.
coverage.strip_path_prefixstring(none)Strip this prefix from coverage file paths. Required for Docker Compose β€” set to the container mount point (e.g., /app).
+ ## Config overrides ### Flags that override config diff --git a/docs/drift/coverage.md b/docs/drift/coverage.md new file mode 100644 index 0000000..9e4f834 --- /dev/null +++ b/docs/drift/coverage.md @@ -0,0 +1,228 @@ +# Code Coverage + +Tusk Drift can collect code coverage during test replay, showing which lines of your service code each trace test exercises. This helps you understand test value, identify redundant tests, and measure suite completeness. + +Coverage works with Node.js (JavaScript, TypeScript, ESM, CJS) and Python. + +## Enabling Coverage + +There are two ways to enable coverage: + +### Config-driven (for CI) + +Add `coverage.enabled: true` to `.tusk/config.yaml`. Coverage is automatically collected during validation runs on the default branch β€” no CI changes needed. + +```yaml +coverage: + enabled: true +``` + +Config-driven coverage is **silent** β€” no console output. Data is collected for backend upload during suite validation. + +### Flag-driven (for local dev) + +```bash +# Show coverage in console +tusk drift run --show-coverage --print + +# Export to file (implies coverage collection) +tusk drift run --coverage-output coverage.lcov --print +``` + +## CLI Flags + +| Flag | Description | +|------|-------------| +| `--show-coverage` | Collect and display code coverage. Forces concurrency to 1. | +| `--coverage-output ` | Write coverage data to a file. LCOV format by default; JSON if path ends in `.json`. Implies coverage collection. | + +### When coverage activates + +| Scenario | Coverage collected? | Shown in console? | +|---|---|---| +| `coverage.enabled: true` + validation run (CI) | Yes | No (silent) | +| `coverage.enabled: true` + local/PR run | No | No | +| `--show-coverage` (any context) | Yes | Yes | +| `--coverage-output` (any context) | Yes | Only if `--show-coverage` also set | + +## Configuration + +Optional include/exclude patterns in `.tusk/config.yaml`: + +```yaml +coverage: + include: + - "backend/src/**" # only report on your service's code + exclude: + - "**/migrations/**" # exclude database migrations + - "**/generated/**" # exclude generated code + - "**/*.test.ts" # exclude test files loaded at startup +``` + + + + + + + + + + + + + + + + + + + + + + + + +
KeyTypeDefaultDescription
coverage.includestring[](all files)If set, only files matching at least one pattern are included in coverage reports. Useful for monorepos.
coverage.excludestring[](none)Files matching any pattern are excluded from coverage reports. Applied after include.
+ +### Pattern syntax + +Patterns use glob matching with `**` for recursive directory matching. File paths are **relative to the git root** (e.g., `backend/src/db/migrations/1700-Init.ts`). + +| Pattern | Matches | +|---------|---------| +| `**/migrations/**` | Any file in any `migrations/` directory | +| `backend/src/**` | All files under `backend/src/` | +| `**/*.test.ts` | Any `.test.ts` file | +| `backend/src/db/migrations/**` | Specific subdirectory | +| `migrations/**` | **Won't match** — paths include the full git-relative prefix | + +## Output + +### Console output + +Coverage is displayed in two places during a run: + +**Per-test (inline):** After each test completes, a single line shows how many lines that specific test covered: +``` +NO DEVIATION - dc14ba0733bdba8b65c11f14c6407320 (63ms) + ↳ coverage: 59 lines across 10 files +``` + +**Aggregate (end of run):** After all tests complete, the full summary shows: +``` +πŸ“Š Coverage: 85.9% lines (55/64), 42.9% branches (6/14) across 2 files + + Per-file: + server.js 85.2% (52/61) + tuskDriftInit.js 100.0% (3/3) + + Per-test: + GET /api/random-user 4 lines across 1 files + POST /api/create-post 5 lines across 1 files +``` + +In TUI mode, the aggregate summary appears in the service logs panel after all tests complete. Per-test detail is shown in each test's log panel. + +### LCOV export + +```bash +tusk drift run --show-coverage --coverage-output coverage.lcov +``` + +Standard LCOV format, compatible with Codecov, Coveralls, SonarQube, VS Code coverage gutters, JetBrains, and most other tools. + +**Note on validation runs:** +- **In-suite tests** are always included in coverage output, even if they fail (a failing test still exercises code paths). +- **Draft tests** are excluded from coverage output. Draft coverage data is uploaded to the backend for promotion decisions ("does this draft add unique coverage?"). +- **After promotion**, the Tusk Cloud dashboard may show slightly higher coverage than the LCOV file (newly promoted drafts are included). The LCOV catches up on the next validation run. + +### JSON export + +```bash +tusk drift run --show-coverage --coverage-output coverage.json +``` + +JSON includes three top-level fields: + +- `summary` β€” aggregate stats, per-file percentages, per-test line counts +- `aggregate` β€” line-level hit counts and branch data for every file +- `per_test` β€” per-test per-file covered lines + +```json +{ + "summary": { + "aggregate": { "coverage_pct": 85.9, "total_covered_lines": 55, "total_coverable_lines": 64 }, + "per_file": { "server.js": { "coverage_pct": 85.2, "covered_lines": 52, "coverable_lines": 61 } }, + "per_test": [{ "test_name": "GET /api/random-user", "covered_lines": 4, "files_touched": 1 }] + }, + "aggregate": { + "server.js": { + "lines": { "1": 1, "5": 3, "12": 0 }, + "total_branches": 14, + "covered_branches": 6, + "branches": { "25": { "total": 2, "covered": 1 } } + } + }, + "per_test": { + "trace-id-abc": { + "server.js": { "covered_lines": [5, 15, 22], "covered_count": 3, "files_touched": 1 } + } + } +} +``` + +## How It Works + +1. CLI starts your service with coverage env vars (`NODE_V8_COVERAGE` for Node, `TUSK_COVERAGE` for Python) +2. After the service is ready, CLI takes a **baseline snapshot** β€” all coverable lines (including uncovered) for the denominator +3. After each test, CLI takes a **per-test snapshot** β€” only lines executed since the last snapshot (counters auto-reset) +4. CLI merges per-test data with baseline to compute the aggregate + +Coverage data flows via the existing CLI-SDK protobuf channel. No extra HTTP servers or ports. + +**Node.js:** Uses V8's built-in precise coverage. No external dependencies. TypeScript source maps handled automatically (`sourceMap: true` in tsconfig required). See the [Node SDK coverage docs](https://github.com/Use-Tusk/drift-node-sdk/blob/main/docs/coverage.md) for internals. + +**Python:** Uses `coverage.py` with `branch=True`. Requires `pip install coverage`. See the [Python SDK coverage docs](https://github.com/Use-Tusk/drift-python-sdk/blob/main/docs/coverage.md) for internals. + +## Docker Compose + +For services running in Docker Compose, two things are needed: + +### 1. Pass coverage env vars to the container + +Add to `docker-compose.tusk-override.yml`: + +```yaml +services: + your-service: + environment: + - TUSK_COVERAGE=${TUSK_COVERAGE:-} # pass through from CLI + - NODE_V8_COVERAGE=/tmp/tusk-v8-coverage # Node.js only: fixed container path +``` + +`TUSK_COVERAGE` is passed through from the CLI using `${TUSK_COVERAGE:-}`. `NODE_V8_COVERAGE` must be a **fixed container path** β€” not `${NODE_V8_COVERAGE:-}` β€” because the CLI creates a host temp directory that doesn't exist inside the container. + +**Python containers:** Add `coverage>=7.0` to your `requirements.txt`. No `NODE_V8_COVERAGE` needed. + +### 2. Strip container path prefix + +Coverage paths from Docker are container-absolute (e.g., `/app/app/api/views.py`). Use `strip_path_prefix` to convert them to repo-relative paths: + +```yaml +coverage: + enabled: true + strip_path_prefix: "/app" # your Docker volume mount point +``` + +This strips `/app` from all paths, so `/app/app/api/views.py` becomes `app/api/views.py` β€” matching the file path in your git repo. Set this to whatever your `docker-compose.yaml` volume mount maps your project root to (e.g., `- .:/app` β†’ use `/app`). + +## Limitations + +- **Concurrency forced to 1.** Per-test snapshots rely on counter resets between tests. +- **Only loaded files tracked.** Files never imported by the server (standalone scripts, test files, unused utils) don't appear in coverage. The denominator only includes files V8/Python actually loaded. +- **Startup code inflates coverage.** Module loading, decorator execution, and DI registration all count as "covered lines." A single test may show 20%+ coverage on a large app from startup alone. +- **TypeScript compiled output.** If using `tsc`, ensure a clean build (`rm -rf dist && tsc`) to avoid stale artifacts with broken imports. +- **Multi-process servers.** Node cluster mode and gunicorn with multiple workers need single-process mode for coverage. +- **Python overhead.** `coverage.py` adds 10-30% execution overhead via `sys.settrace()`. V8 coverage is near-zero overhead. +- **Python branch coverage** uses a private coverage.py API (`_analyze()`). May break on major coverage.py upgrades. +- **Docker paths.** Coverage paths are container-absolute by default. Use `coverage.strip_path_prefix` to convert to repo-relative paths (see Docker Compose section above). From 757626ec65d431e9e8e92f1295c3201e84021529 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Fri, 3 Apr 2026 14:40:40 -0700 Subject: [PATCH 21/36] feat: config-driven coverage activation and code quality fixes Coverage activation: - coverage.enabled config option for automatic collection during validation - --show-coverage flag replaces --coverage (display only, for local dev) - Config-driven mode is silent (no console output), for CI upload - coverage.strip_path_prefix for Docker container path normalization Code quality: - Add coverageBaselineMu mutex to prevent races - Recompute branch totals in SetCoverageBaseline from merged per-line data - Clean up V8 temp directory in StopService - Return map copy from GetTestCoverageDetail (prevent concurrent access) - Deduplicate formatCoverageSummary (shared by print and TUI) - Simplify printCoverageSummary (no error return) - Remove redundant CoverageFileExport type (use FileCoverageData with JSON tags) - Filter per-test data by include/exclude in JSON export - Filter to in-suite records for aggregate (drafts excluded from output) - Use doublestar for glob matching (replace hand-rolled implementation) - Fix TUI path shortening (only Rel() on absolute paths) - Sort files alphabetically, use DisplayName for GraphQL tests - Remove duplicate coverageRecords (single source in executor) - Add strip_path_prefix tests --- cmd/run.go | 75 +++++-- internal/config/config.go | 8 + internal/runner/coverage.go | 344 ++++++++++++++++++++++++++++--- internal/runner/coverage_test.go | 124 ++++++++++- internal/runner/executor.go | 74 ++++++- internal/runner/server.go | 2 +- internal/runner/service.go | 6 + internal/tui/test_executor.go | 42 +++- 8 files changed, 610 insertions(+), 65 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 616cde8..dcfa9a7 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -57,7 +57,8 @@ var ( validateSuite bool // Coverage mode - coverageEnabled bool + showCoverage bool + coverageOutputPath string ) //go:embed short_docs/drift/drift_run.md @@ -119,7 +120,8 @@ func bindRunFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&validateSuite, "validate-suite", false, "[Cloud] Force validation mode regardless of branch") // Coverage mode - cmd.Flags().BoolVar(&coverageEnabled, "coverage", false, "Collect code coverage during test execution") + cmd.Flags().BoolVar(&showCoverage, "show-coverage", false, "Collect and display code coverage during test execution") + cmd.Flags().StringVar(&coverageOutputPath, "coverage-output", "", "Write coverage data to file (LCOV by default, JSON if path ends in .json)") _ = cmd.Flags().MarkHidden("client-id") cmd.Flags().SortFlags = false @@ -309,12 +311,38 @@ func runTests(cmd *cobra.Command, args []string) error { } executor.SetEnableServiceLogs(enableServiceLogs || debug) + + // Coverage activation: + // - Config-driven: coverage.enabled=true in config activates during validation runs (silent, for upload) + // - Flag-driven: --show-coverage or --coverage-output activates anytime (for local dev/debugging) + coverageFromConfig := getConfigErr == nil && cfg.Coverage.Enabled && isValidation + coverageFromFlags := showCoverage || coverageOutputPath != "" + coverageEnabled := coverageFromConfig || coverageFromFlags if coverageEnabled { executor.SetCoverageEnabled(true) + executor.SetShowCoverage(showCoverage) + if coverageOutputPath != "" { + executor.SetCoverageOutputPath(coverageOutputPath) + } + if getConfigErr == nil { + if len(cfg.Coverage.Include) > 0 { + executor.SetCoverageIncludePatterns(cfg.Coverage.Include) + } + if len(cfg.Coverage.Exclude) > 0 { + executor.SetCoverageExcludePatterns(cfg.Coverage.Exclude) + } + if cfg.Coverage.StripPathPrefix != "" { + executor.SetCoverageStripPrefix(cfg.Coverage.StripPathPrefix) + } + } // Coverage requires serial execution (concurrency=1) because per-test // snapshots rely on the SDK resetting counters between tests. executor.SetConcurrency(1) - log.Stderrln("➀ Coverage collection enabled (concurrency forced to 1)") + if showCoverage { + log.Stderrln("➀ Coverage collection enabled (concurrency forced to 1)") + } else { + log.Debug("Coverage collection enabled via config (concurrency forced to 1)") + } } if saveResults { if resultsDir == "" { @@ -370,6 +398,9 @@ func runTests(cmd *cobra.Command, args []string) error { res, test, ) + // TODO-COVERAGE-TRACKING: Include per-test coverage in the upload. + // Add TraceTestCoverageData to UploadSingleTestResult (or piggyback on existing proto). + // Data source: executor.GetTestCoverageDetail(test.TraceID) mu.Lock() attemptedCount++ @@ -403,8 +434,6 @@ func runTests(cmd *cobra.Command, args []string) error { } // Coverage: wrap the OnTestCompleted callback to take snapshots between tests - var coverageRecords []runner.CoverageTestRecord - var coverageMu sync.Mutex if coverageEnabled { existingCallback := executor.OnTestCompleted executor.SetOnTestCompleted(func(res runner.TestResult, test runner.Test) { @@ -420,20 +449,19 @@ func runTests(cmd *cobra.Command, args []string) error { return } - coverageMu.Lock() - coverageRecords = append(coverageRecords, runner.CoverageTestRecord{ - TestID: test.TraceID, - TestName: fmt.Sprintf("%s %s", test.Method, test.Path), - Coverage: lineCounts, + executor.AddCoverageRecord(runner.CoverageTestRecord{ + TestID: test.TraceID, + TestName: test.DisplayName, + SuiteStatus: test.SuiteStatus, + Coverage: lineCounts, }) - coverageMu.Unlock() // Store detail for TUI display detail := runner.SnapshotToCoverageDetail(lineCounts) executor.SetTestCoverageDetail(test.TraceID, detail) - // Print sub-line in --print mode - if !interactive { + // Print sub-line in --print mode when --show-coverage is active + if !interactive && showCoverage { totalLines := 0 for _, fd := range detail { totalLines += fd.CoveredCount @@ -921,15 +949,12 @@ func runTests(cmd *cobra.Command, args []string) error { _ = os.Stdout.Sync() time.Sleep(1 * time.Millisecond) - // Coverage: write per-test files and print summary - if coverageEnabled && len(coverageRecords) > 0 { - coverageMu.Lock() - records := make([]runner.CoverageTestRecord, len(coverageRecords)) - copy(records, coverageRecords) - coverageMu.Unlock() - - if err := executor.ProcessCoverage(records); err != nil { - log.Warn("Failed to process coverage", "error", err) + // Coverage: print summary and write output file + if coverageEnabled { + if records := executor.GetCoverageRecords(); len(records) > 0 { + if err := executor.ProcessCoverage(records); err != nil { + log.Warn("Failed to process coverage", "error", err) + } } } @@ -959,6 +984,12 @@ func runTests(cmd *cobra.Command, args []string) error { if isValidation { log.Println("\nSuite validation completed - backend will process results and update suite") } + // TODO-COVERAGE-TRACKING: Upload coverage baseline after validation run completes. + // When coverageEnabled && isValidation, upload the baseline snapshot to backend: + // - endpoint: UploadCoverageBaseline (new) + // - data: executor.coverageBaseline (all coverable lines + branch data) + // - identifiers: driftRunID, observable_service_id, commit_sha + // This provides the denominator for coverage % calculations. mu.Lock() log.Stderr(fmt.Sprintf("\nSuccessfully uploaded %d/%d test results", uploadedCount, attemptedCount)) if attemptedCount > uploadedCount && lastUploadErr != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 9cffb2f..be1dc91 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,7 @@ type Config struct { Replay ReplayConfig `koanf:"replay"` Traces TracesConfig `koanf:"traces"` Results ResultsConfig `koanf:"results"` + Coverage CoverageConfig `koanf:"coverage"` } type ServiceConfig struct { @@ -115,6 +116,13 @@ type ResultsConfig struct { Dir string `koanf:"dir"` } +type CoverageConfig struct { + Enabled bool `koanf:"enabled"` + Include []string `koanf:"include"` + Exclude []string `koanf:"exclude"` + StripPathPrefix string `koanf:"strip_path_prefix"` +} + // Load loads the config file and applies environment overrides. // This function is idempotent - calling it multiple times will only load once. func Load(configFile string) error { diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index a359ae9..7893261 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -1,6 +1,7 @@ package runner import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -11,12 +12,13 @@ import ( "github.com/Use-Tusk/tusk-cli/internal/log" "github.com/Use-Tusk/tusk-cli/internal/utils" + "github.com/bmatcuk/doublestar/v4" ) const ( coverageBaselineMaxRetries = 15 coverageBaselineRetryDelay = 200 * time.Millisecond - coverageSnapshotTimeout = 10 * time.Second + coverageSnapshotTimeout = 30 * time.Second ) // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. @@ -75,7 +77,7 @@ func (e *Executor) callCoverageEndpoint(baseline bool) (CoverageSnapshot, error) } } - return normalizeCoveragePaths(snapshot), nil + return normalizeCoveragePaths(snapshot, e.coverageStripPrefix), nil } // BranchInfo tracks branch coverage at a specific line. @@ -86,10 +88,10 @@ type BranchInfo struct { // FileCoverageData is the internal representation of per-file coverage. type FileCoverageData struct { - Lines map[string]int - TotalBranches int - CoveredBranches int - Branches map[string]BranchInfo + Lines map[string]int `json:"lines"` + TotalBranches int `json:"total_branches"` + CoveredBranches int `json:"covered_branches"` + Branches map[string]BranchInfo `json:"branches,omitempty"` } // CoverageSnapshot is the full coverage data for a snapshot. @@ -97,9 +99,10 @@ type CoverageSnapshot map[string]FileCoverageData // CoverageTestRecord holds per-test coverage data. type CoverageTestRecord struct { - TestID string - TestName string - Coverage CoverageSnapshot + TestID string + TestName string + SuiteStatus string // "draft", "in_suite", or "" (local) + Coverage CoverageSnapshot } // CoverageFileDiff represents per-test coverage for a single file. @@ -148,20 +151,76 @@ func SnapshotToCoverageDetail(snapshot CoverageSnapshot) map[string]CoverageFile return result } -// ProcessCoverage computes aggregate coverage and prints the summary. -// All data stays in memory - no files written to user's project. +// ProcessCoverage computes aggregate coverage, optionally prints summary, writes file, and prepares for upload. +// During validation runs, the aggregate and output files only include IN_SUITE tests (not drafts). +// All per-test data (including drafts) is retained for backend upload β€” the backend needs draft +// coverage for promotion decisions ("does this draft add unique coverage?"). func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { if !e.coverageEnabled || len(records) == 0 { return nil } - log.Stderrln("\n➀ Processing coverage data...") + // Filter to in-suite tests for the aggregate. If no suite status is set + // (local run, no cloud), include all tests. + suiteRecords := filterInSuiteRecords(records) // Compute aggregate: start with baseline (all coverable lines including count=0), // then merge in per-test coverage. This gives accurate denominator. - aggregate := mergeWithBaseline(e.coverageBaseline, records) + aggregate := mergeWithBaseline(e.coverageBaseline, suiteRecords) + + // Apply include/exclude patterns from config + aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + + // Print summary if --show-coverage was passed (not in silent config-driven mode) + if e.coverageShowOutput { + log.Stderrln("\n➀ Processing coverage data...") + e.printCoverageSummary(suiteRecords, aggregate) + } + + // Write coverage file if requested. + // During validation runs, aggregate and output only include IN_SUITE tests. + // Draft coverage is excluded from the file but retained for backend upload. + if e.coverageOutputPath != "" { + outPath := e.coverageOutputPath + if !filepath.IsAbs(outPath) { + if cwd, err := os.Getwd(); err == nil { + outPath = filepath.Join(cwd, outPath) + } + } + if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { + return fmt.Errorf("failed to create coverage output directory: %w", err) + } + + if strings.HasSuffix(strings.ToLower(outPath), ".json") { + if err := WriteCoverageJSON(outPath, aggregate, e.coveragePerTest, suiteRecords); err != nil { + return fmt.Errorf("failed to write coverage JSON: %w", err) + } + } else { + if err := WriteCoverageLCOV(outPath, aggregate); err != nil { + return fmt.Errorf("failed to write coverage LCOV: %w", err) + } + } + if e.coverageShowOutput { + log.Stderrln(fmt.Sprintf("\nπŸ“„ Coverage written to %s", e.coverageOutputPath)) + } + } + + // TODO-COVERAGE-TRACKING: Upload coverage data to backend during validation runs. + // Upload ALL records (including drafts) β€” backend needs draft coverage for promotion. + // The backend computes the in-suite aggregate itself after promotion decisions. + // When running with --ci and isValidation=true, upload: + // 1. Coverage baseline (all coverable lines) β€” one per validation run + // 2. Per-test coverage records β€” one per trace test (both IN_SUITE and DRAFT) + // Backend uses this for: + // - Coverage-aware test promotion (which drafts add unique coverage?) + // - Coverage history tracking (coverage_snapshot table, IN_SUITE only) + // - Dashboard display (per-file, per-test attribution) + // Backend computes the final aggregate from IN_SUITE tests ONLY (post-promotion). + // The per-test data for rejected drafts is stored but excluded from the aggregate. + // Upload should use existing UploadTraceTestResults piggybacking (add TraceTestCoverageData) + // and a new UploadCoverageBaseline endpoint. - return e.printCoverageSummary(records, aggregate) + return nil } // mergeWithBaseline creates aggregate coverage by starting from the baseline @@ -354,12 +413,12 @@ func ComputeCoverageSummary( return summary } -// printCoverageSummary computes and prints the coverage summary to stderr. -func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) error { - summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) +// formatCoverageSummary formats coverage summary as lines of text. +func (e *Executor) formatCoverageSummary(summary CoverageSummary) []string { + var lines []string // Aggregate line - coverageMsg := fmt.Sprintf("\nπŸ“Š Coverage: %.1f%% lines (%d/%d)", + coverageMsg := fmt.Sprintf("πŸ“Š Coverage: %.1f%% lines (%d/%d)", summary.Aggregate.CoveragePct, summary.Aggregate.TotalCoveredLines, summary.Aggregate.TotalCoverableLines) if summary.Aggregate.TotalBranches > 0 { coverageMsg += fmt.Sprintf(", %.1f%% branches (%d/%d)", @@ -369,9 +428,9 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate if e.coverageBaseline == nil { coverageMsg += " ⚠️ baseline failed - denominator may be incomplete" } - log.Stderrln(coverageMsg) + lines = append(lines, coverageMsg) - // Per-file breakdown sorted by coverage % + // Per-file breakdown sorted alphabetically type fileStat struct { path string pct float64 @@ -383,11 +442,22 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate stats = append(stats, fileStat{fp, fs.CoveragePct, fs.CoveredLines, fs.CoverableLines}) } } - sort.Slice(stats, func(i, j int) bool { return stats[i].pct > stats[j].pct }) + sort.Slice(stats, func(i, j int) bool { return stats[i].path < stats[j].path }) - log.Stderrln("\n Per-file:") + lines = append(lines, "") + lines = append(lines, " Per-file:") for _, s := range stats { - log.Stderrln(fmt.Sprintf(" %-40s %5.1f%% (%d/%d)", s.path, s.pct, s.cov, s.tot)) + lines = append(lines, fmt.Sprintf(" %-40s %5.1f%% (%d/%d)", s.path, s.pct, s.cov, s.tot)) + } + + return lines +} + +// printCoverageSummary computes and prints the coverage summary to stderr. +func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) { + summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) + for _, line := range e.formatCoverageSummary(summary) { + log.Stderrln(line) } // Per-test breakdown @@ -399,8 +469,203 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate } log.Stderrln(fmt.Sprintf(" %-40s %d lines across %d files", name, ts.CoveredLines, ts.FilesTouched)) } +} - return nil +// FormatCoverageSummaryLines computes the coverage summary and returns formatted +// lines for the TUI service log panel (aggregate + per-file, no per-test). +func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) []string { + if !e.coverageEnabled || len(records) == 0 { + return nil + } + + aggregate := mergeWithBaseline(e.coverageBaseline, records) + aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) + return e.formatCoverageSummary(summary) +} + +// filterCoverageByPatterns applies include/exclude glob patterns to a snapshot. +// Include (if set): only keep files matching at least one include pattern. +// Exclude: remove files matching any exclude pattern. +// Include is applied first, then exclude. +// Supports ** for recursive directory matching: +// - "**/migrations/**" matches any file in any migrations/ directory +// - "backend/src/db/**" matches everything under backend/src/db/ +// - "**/*.test.ts" matches any .test.ts file +// - "backend/src/db/migrations/**" matches specific path +func filterCoverageByPatterns(snapshot CoverageSnapshot, include, exclude []string) CoverageSnapshot { + if len(include) == 0 && len(exclude) == 0 { + return snapshot + } + filtered := make(CoverageSnapshot, len(snapshot)) + for filePath, data := range snapshot { + // Include filter: if patterns are set, file must match at least one + if len(include) > 0 && !matchesAnyPattern(filePath, include) { + continue + } + // Exclude filter: file must not match any + if len(exclude) > 0 && matchesAnyPattern(filePath, exclude) { + continue + } + filtered[filePath] = data + } + return filtered +} + +// matchesAnyPattern checks if a file path matches any of the glob patterns. +// Uses doublestar for proper ** support. +func matchesAnyPattern(filePath string, patterns []string) bool { + for _, pattern := range patterns { + if matched, _ := doublestar.Match(pattern, filePath); matched { + return true + } + } + return false +} + +// matchGlob matches a path against a glob pattern supporting **. +// Exported for testing. +func matchGlob(filePath, pattern string) bool { + matched, _ := doublestar.Match(pattern, filePath) + return matched +} + +// --- Coverage Export --- + +// CoverageExport is the top-level JSON export structure. +type CoverageExport struct { + Summary CoverageSummary `json:"summary"` + Aggregate CoverageSnapshot `json:"aggregate"` + PerTest map[string]map[string]CoverageFileDiff `json:"per_test"` +} + +// WriteCoverageJSON writes aggregate + per-test coverage as JSON. +func WriteCoverageJSON(path string, aggregate CoverageSnapshot, perTest map[string]map[string]CoverageFileDiff, records []CoverageTestRecord) error { + summary := ComputeCoverageSummary(aggregate, perTest, records) + + // Filter per-test data to only include files present in the (filtered) aggregate + filteredPerTest := make(map[string]map[string]CoverageFileDiff, len(perTest)) + for testID, testDetail := range perTest { + filtered := make(map[string]CoverageFileDiff) + for fp, fd := range testDetail { + if _, ok := aggregate[fp]; ok { + filtered[fp] = fd + } + } + if len(filtered) > 0 { + filteredPerTest[testID] = filtered + } + } + + export := CoverageExport{ + Summary: summary, + Aggregate: aggregate, + PerTest: filteredPerTest, + } + + data, err := json.MarshalIndent(export, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, data, 0644) +} + +// WriteCoverageLCOV writes aggregate coverage data in LCOV format. +func WriteCoverageLCOV(path string, aggregate CoverageSnapshot) error { + var b strings.Builder + + // Sort file paths for deterministic output + filePaths := make([]string, 0, len(aggregate)) + for fp := range aggregate { + filePaths = append(filePaths, fp) + } + sort.Strings(filePaths) + + for _, filePath := range filePaths { + fileData := aggregate[filePath] + b.WriteString("SF:") + b.WriteString(filePath) + b.WriteByte('\n') + + // Line data (DA:line,count) + lineNums := make([]int, 0, len(fileData.Lines)) + for lineStr := range fileData.Lines { + if n, err := strconv.Atoi(lineStr); err == nil { + lineNums = append(lineNums, n) + } + } + sort.Ints(lineNums) + + linesFound := 0 + linesHit := 0 + for _, line := range lineNums { + count := fileData.Lines[strconv.Itoa(line)] + b.WriteString(fmt.Sprintf("DA:%d,%d\n", line, count)) + linesFound++ + if count > 0 { + linesHit++ + } + } + + // Branch data (BRDA:line,block,branch,count) + branchLines := make([]int, 0, len(fileData.Branches)) + for lineStr := range fileData.Branches { + if n, err := strconv.Atoi(lineStr); err == nil { + branchLines = append(branchLines, n) + } + } + sort.Ints(branchLines) + + branchesFound := 0 + branchesHit := 0 + for _, line := range branchLines { + info := fileData.Branches[strconv.Itoa(line)] + for i := 0; i < info.Total; i++ { + count := 0 + if i < info.Covered { + count = 1 + } + b.WriteString(fmt.Sprintf("BRDA:%d,0,%d,%d\n", line, i, count)) + branchesFound++ + if count > 0 { + branchesHit++ + } + } + } + + b.WriteString(fmt.Sprintf("LF:%d\n", linesFound)) + b.WriteString(fmt.Sprintf("LH:%d\n", linesHit)) + if branchesFound > 0 { + b.WriteString(fmt.Sprintf("BRF:%d\n", branchesFound)) + b.WriteString(fmt.Sprintf("BRH:%d\n", branchesHit)) + } + b.WriteString("end_of_record\n") + } + + return os.WriteFile(path, []byte(b.String()), 0644) +} + +// filterInSuiteRecords returns only records from in-suite tests. +// If no tests have suite status set (local run, no cloud), returns all records. +func filterInSuiteRecords(records []CoverageTestRecord) []CoverageTestRecord { + hasSuiteStatus := false + for _, r := range records { + if r.SuiteStatus != "" { + hasSuiteStatus = true + break + } + } + if !hasSuiteStatus { + return records + } + + var filtered []CoverageTestRecord + for _, r := range records { + if r.SuiteStatus != "draft" { + filtered = append(filtered, r) + } + } + return filtered } // --- Helpers --- @@ -420,10 +685,32 @@ func dedup(sorted []int) []int { // normalizeCoveragePaths converts absolute file paths to repo-relative paths. -// Uses git root as the base (consistent across machines, handles monorepo -// files outside the service directory like ../shared/utils.js). -// Falls back to cwd if not in a git repo. -func normalizeCoveragePaths(snapshot CoverageSnapshot) CoverageSnapshot { +// +// For non-Docker: uses git root as the base (handles monorepos, cd into subdirs). +// For Docker: coverage.strip_path_prefix strips the container mount point first +// (e.g., "/app"), then git root normalization converts the rest to repo-relative. +func normalizeCoveragePaths(snapshot CoverageSnapshot, stripPrefix string) CoverageSnapshot { + if len(snapshot) == 0 { + return snapshot + } + + // Step 1: Strip container path prefix if configured (Docker Compose) + if stripPrefix != "" { + stripPrefix = strings.TrimRight(stripPrefix, "/") + stripped := make(CoverageSnapshot, len(snapshot)) + for absPath, fileData := range snapshot { + newPath := absPath + if strings.HasPrefix(absPath, stripPrefix+"/") { + newPath = absPath[len(stripPrefix)+1:] + } else if absPath == stripPrefix { + newPath = "." + } + stripped[newPath] = fileData + } + snapshot = stripped + } + + // Step 2: Normalize to git-root-relative paths base := getPathNormalizationBase() if base == "" { return snapshot @@ -433,6 +720,7 @@ func normalizeCoveragePaths(snapshot CoverageSnapshot) CoverageSnapshot { for absPath, fileData := range snapshot { relPath, err := filepath.Rel(base, absPath) if err != nil || strings.HasPrefix(relPath, "..") { + // Already relative (from strip_prefix) or outside git root β€” keep as-is relPath = absPath } normalized[relPath] = fileData diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go index f1ab6ea..110983b 100644 --- a/internal/runner/coverage_test.go +++ b/internal/runner/coverage_test.go @@ -295,15 +295,133 @@ func TestComputeCoverageSummary(t *testing.T) { func TestNormalizeCoveragePaths(t *testing.T) { t.Run("nil input returns empty", func(t *testing.T) { - result := normalizeCoveragePaths(nil) + result := normalizeCoveragePaths(nil, "") assert.Len(t, result, 0) }) t.Run("empty input returns empty", func(t *testing.T) { - result := normalizeCoveragePaths(CoverageSnapshot{}) + result := normalizeCoveragePaths(CoverageSnapshot{}, "") assert.Empty(t, result) }) - // Note: full path normalization depends on git root which is environment-specific. + t.Run("strip_path_prefix strips container mount point", func(t *testing.T) { + snapshot := CoverageSnapshot{ + "/app/app/api/views.py": FileCoverageData{Lines: map[string]int{"1": 1}}, + "/app/app/settings.py": FileCoverageData{Lines: map[string]int{"1": 1}}, + "/app/tusk_drift_init.py": FileCoverageData{Lines: map[string]int{"1": 1}}, + } + result := normalizeCoveragePaths(snapshot, "/app") + assert.Contains(t, result, "app/api/views.py") + assert.Contains(t, result, "app/settings.py") + assert.Contains(t, result, "tusk_drift_init.py") + }) + + t.Run("strip_path_prefix with trailing slash", func(t *testing.T) { + snapshot := CoverageSnapshot{ + "/app/server.py": FileCoverageData{Lines: map[string]int{"1": 1}}, + } + result := normalizeCoveragePaths(snapshot, "/app/") + assert.Contains(t, result, "server.py") + }) + + t.Run("strip_path_prefix with cd backend", func(t *testing.T) { + snapshot := CoverageSnapshot{ + "/app/backend/src/server.py": FileCoverageData{Lines: map[string]int{"1": 1}}, + } + result := normalizeCoveragePaths(snapshot, "/app") + assert.Contains(t, result, "backend/src/server.py") + }) + + // Note: full git root normalization depends on git root which is environment-specific. // We test the function handles edge cases; full integration is tested E2E. } + + +func TestMatchGlob(t *testing.T) { + tests := []struct { + path string + pattern string + want bool + }{ + // ** patterns + {"backend/src/db/migrations/1700-Init.ts", "**/migrations/**", true}, + {"backend/src/db/migrations/foo/bar.ts", "**/migrations/**", true}, + {"backend/src/services/ResourceService.ts", "**/migrations/**", false}, + + // Leading ** + {"backend/src/utils/test.test.ts", "**/*.test.ts", true}, + {"foo.test.ts", "**/*.test.ts", true}, + {"backend/src/utils/test.ts", "**/*.test.ts", false}, + + // Trailing ** + {"backend/src/db/migrations/1700-Init.ts", "backend/src/db/**", true}, + {"backend/src/db/config.ts", "backend/src/db/**", true}, + {"backend/src/services/foo.ts", "backend/src/db/**", false}, + + // Specific path with ** + {"backend/src/db/migrations/1700-Init.ts", "backend/src/db/migrations/**", true}, + {"backend/src/db/config.ts", "backend/src/db/migrations/**", false}, + + // No ** β€” standard glob + {"server.js", "server.js", true}, + {"server.ts", "server.js", false}, + {"server.js", "*.js", true}, + } + + for _, tt := range tests { + t.Run(tt.path+"_"+tt.pattern, func(t *testing.T) { + got := matchGlob(tt.path, tt.pattern) + assert.Equal(t, tt.want, got, "matchGlob(%q, %q)", tt.path, tt.pattern) + }) + } +} + +func TestFilterCoverageByPatterns(t *testing.T) { + snapshot := CoverageSnapshot{ + "backend/src/db/migrations/1700-Init.ts": FileCoverageData{Lines: map[string]int{"1": 1}}, + "backend/src/db/migrations/1701-Add.ts": FileCoverageData{Lines: map[string]int{"1": 1}}, + "backend/src/services/ResourceService.ts": FileCoverageData{Lines: map[string]int{"1": 1}}, + "backend/src/scripts/runMigration.ts": FileCoverageData{Lines: map[string]int{"1": 1}}, + "backend/src/utils/test.test.ts": FileCoverageData{Lines: map[string]int{"1": 1}}, + "shared/utils/helpers.ts": FileCoverageData{Lines: map[string]int{"1": 1}}, + } + + t.Run("exclude only", func(t *testing.T) { + result := filterCoverageByPatterns(snapshot, nil, []string{ + "**/migrations/**", + "**/scripts/**", + }) + assert.Len(t, result, 3) + assert.Contains(t, result, "backend/src/services/ResourceService.ts") + assert.Contains(t, result, "backend/src/utils/test.test.ts") + assert.Contains(t, result, "shared/utils/helpers.ts") + }) + + t.Run("include only", func(t *testing.T) { + result := filterCoverageByPatterns(snapshot, []string{ + "backend/src/**", + }, nil) + assert.Len(t, result, 5) + assert.Contains(t, result, "backend/src/services/ResourceService.ts") + assert.NotContains(t, result, "shared/utils/helpers.ts") + }) + + t.Run("include and exclude", func(t *testing.T) { + result := filterCoverageByPatterns(snapshot, []string{ + "backend/src/**", + }, []string{ + "**/migrations/**", + }) + assert.Len(t, result, 3) + assert.Contains(t, result, "backend/src/services/ResourceService.ts") + assert.Contains(t, result, "backend/src/scripts/runMigration.ts") + assert.Contains(t, result, "backend/src/utils/test.test.ts") + assert.NotContains(t, result, "shared/utils/helpers.ts") + assert.NotContains(t, result, "backend/src/db/migrations/1700-Init.ts") + }) + + t.Run("no patterns returns all", func(t *testing.T) { + result := filterCoverageByPatterns(snapshot, nil, nil) + assert.Len(t, result, 6) + }) +} diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 6db5fe5..b46a59f 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -94,10 +94,19 @@ type Executor struct { replaySandboxConfigPath string // Coverage - coverageEnabled bool + coverageEnabled bool + coverageShowOutput bool + coverageOutputPath string + coverageTempDir string + coverageIncludePatterns []string + coverageExcludePatterns []string + coverageStripPrefix string coveragePerTest map[string]map[string]CoverageFileDiff coveragePerTestMu sync.Mutex coverageBaseline CoverageSnapshot + coverageBaselineMu sync.Mutex + coverageRecords []CoverageTestRecord + coverageRecordsMu sync.Mutex } func NewExecutor() *Executor { @@ -489,9 +498,31 @@ func (e *Executor) IsCoverageEnabled() bool { return e.coverageEnabled } +func (e *Executor) SetShowCoverage(show bool) { + e.coverageShowOutput = show +} + +func (e *Executor) SetCoverageOutputPath(path string) { + e.coverageOutputPath = path +} + +func (e *Executor) SetCoverageIncludePatterns(patterns []string) { + e.coverageIncludePatterns = patterns +} + +func (e *Executor) SetCoverageExcludePatterns(patterns []string) { + e.coverageExcludePatterns = patterns +} + +func (e *Executor) SetCoverageStripPrefix(prefix string) { + e.coverageStripPrefix = prefix +} + // SetCoverageBaseline merges new baseline data into the existing baseline. // Called per environment group - accumulates across service restarts. func (e *Executor) SetCoverageBaseline(baseline CoverageSnapshot) { + e.coverageBaselineMu.Lock() + defer e.coverageBaselineMu.Unlock() if e.coverageBaseline == nil { e.coverageBaseline = make(CoverageSnapshot) } @@ -508,7 +539,7 @@ func (e *Executor) SetCoverageBaseline(baseline CoverageSnapshot) { existing.Lines[line] = count } } - // Merge branch data (keep max) + // Merge branch data (keep max per line) for line, branchInfo := range fileData.Branches { if existing.Branches == nil { existing.Branches = make(map[string]BranchInfo) @@ -517,8 +548,14 @@ func (e *Executor) SetCoverageBaseline(baseline CoverageSnapshot) { existing.Branches[line] = branchInfo } } - existing.TotalBranches = fileData.TotalBranches - existing.CoveredBranches = fileData.CoveredBranches + // Recompute file-level totals from merged per-line data + totalB, covB := 0, 0 + for _, b := range existing.Branches { + totalB += b.Total + covB += b.Covered + } + existing.TotalBranches = totalB + existing.CoveredBranches = covB e.coverageBaseline[filePath] = existing } } @@ -533,14 +570,39 @@ func (e *Executor) SetTestCoverageDetail(testID string, detail map[string]Covera e.coveragePerTest[testID] = detail } -// GetTestCoverageDetail returns per-test coverage diff for a given test. +// GetTestCoverageDetail returns a copy of per-test coverage diff for a given test. func (e *Executor) GetTestCoverageDetail(testID string) map[string]CoverageFileDiff { e.coveragePerTestMu.Lock() defer e.coveragePerTestMu.Unlock() if e.coveragePerTest == nil { return nil } - return e.coveragePerTest[testID] + original := e.coveragePerTest[testID] + if original == nil { + return nil + } + // Return a copy to avoid concurrent map access from TUI goroutines + copied := make(map[string]CoverageFileDiff, len(original)) + for k, v := range original { + copied[k] = v + } + return copied +} + +// AddCoverageRecord stores a per-test coverage record. +func (e *Executor) AddCoverageRecord(record CoverageTestRecord) { + e.coverageRecordsMu.Lock() + defer e.coverageRecordsMu.Unlock() + e.coverageRecords = append(e.coverageRecords, record) +} + +// GetCoverageRecords returns a copy of all coverage records. +func (e *Executor) GetCoverageRecords() []CoverageTestRecord { + e.coverageRecordsMu.Lock() + defer e.coverageRecordsMu.Unlock() + records := make([]CoverageTestRecord, len(e.coverageRecords)) + copy(records, e.coverageRecords) + return records } func (e *Executor) SetSuiteSpans(spans []*core.Span) { diff --git a/internal/runner/server.go b/internal/runner/server.go index b17dc6e..3eb4b24 100644 --- a/internal/runner/server.go +++ b/internal/runner/server.go @@ -1012,7 +1012,7 @@ func (ms *Server) SendCoverageSnapshot(baseline bool) (*core.CoverageSnapshotRes return nil, fmt.Errorf("failed to send coverage snapshot request: %w", err) } - response, err := ms.waitForSDKResponse(requestID, 10*time.Second) + response, err := ms.waitForSDKResponse(requestID, coverageSnapshotTimeout) if err != nil { return nil, fmt.Errorf("failed to receive coverage snapshot response: %w", err) } diff --git a/internal/runner/service.go b/internal/runner/service.go index c1b69fe..7d02859 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -163,6 +163,7 @@ func (e *Executor) StartService() error { if err != nil { return fmt.Errorf("failed to create temp dir for V8 coverage: %w", err) } + e.coverageTempDir = v8CoverageDir env = append(env, fmt.Sprintf("NODE_V8_COVERAGE=%s", v8CoverageDir)) env = append(env, "TS_NODE_EMIT=true") log.Debug("Coverage enabled", "v8_dir", v8CoverageDir) @@ -333,6 +334,11 @@ func (e *Executor) StopService() error { e.fenceManager.Cleanup() e.fenceManager = nil } + // Clean up V8 coverage temp directory + if e.coverageTempDir != "" { + os.RemoveAll(e.coverageTempDir) + e.coverageTempDir = "" + } log.ServiceLog("Service stopped") }() diff --git a/internal/tui/test_executor.go b/internal/tui/test_executor.go index 06975b8..076344e 100644 --- a/internal/tui/test_executor.go +++ b/internal/tui/test_executor.go @@ -630,12 +630,22 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { totalLines += fd.CoveredCount } m.addTestLog(test.TraceID, fmt.Sprintf(" πŸ“Š Coverage: %d lines across %d files", totalLines, len(detail))) - for filePath, fd := range detail { - // Shorten the file path relative to cwd + // Sort file paths for deterministic display + filePaths := make([]string, 0, len(detail)) + for fp := range detail { + filePaths = append(filePaths, fp) + } + slices.Sort(filePaths) + for _, filePath := range filePaths { + fd := detail[filePath] + // Paths are already git-relative from normalizeCoveragePaths. + // Only try Rel() on absolute paths (shouldn't happen, but defensive). shortPath := filePath - if cwd, err := os.Getwd(); err == nil { - if rel, err := filepath.Rel(cwd, filePath); err == nil { - shortPath = rel + if filepath.IsAbs(filePath) { + if cwd, err := os.Getwd(); err == nil { + if rel, err := filepath.Rel(cwd, filePath); err == nil { + shortPath = rel + } } } m.addTestLog(test.TraceID, fmt.Sprintf(" %-40s %d lines", shortPath, fd.CoveredCount)) @@ -740,6 +750,17 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.addServiceLog("\n" + strings.Repeat("=", 60)) m.addServiceLog("🏁 All tests completed!") + // Show aggregate coverage summary in service logs + if m.executor.IsCoverageEnabled() { + records := m.executor.GetCoverageRecords() + if summaryLines := m.executor.FormatCoverageSummaryLines(records); len(summaryLines) > 0 { + m.addServiceLog("") + for _, line := range summaryLines { + m.addServiceLog(line) + } + } + } + // All-tests completed upload (non-blocking) if m.opts != nil && m.opts.OnAllCompleted != nil { results := make([]runner.TestResult, len(m.results)) @@ -1068,6 +1089,17 @@ func (m *testExecutorModel) startNextEnvironmentGroup() tea.Cmd { m.serviceStarted = true m.addServiceLog("βœ… Environment ready") + // Coverage: take baseline snapshot to capture all coverable lines and reset counters + if m.executor.IsCoverageEnabled() { + baseline, err := m.executor.TakeCoverageBaseline() + if err != nil { + m.addServiceLog("⚠️ Coverage baseline failed: " + err.Error()) + } else { + m.executor.SetCoverageBaseline(baseline) + m.addServiceLog("βœ… Coverage baseline captured") + } + } + // Build list of global test indices for this environment envIdx := m.currentGroupIndex - 1 // We already incremented it above m.currentEnvTestIndices = make([]int, 0, len(group.Tests)) From 8d2c4c6279f87d4b624f61cdf8662e188d3d6f70 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Fri, 3 Apr 2026 15:02:32 -0700 Subject: [PATCH 22/36] docs: clean up AI writing patterns in coverage doc --- docs/drift/coverage.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/drift/coverage.md b/docs/drift/coverage.md index 9e4f834..e5180cb 100644 --- a/docs/drift/coverage.md +++ b/docs/drift/coverage.md @@ -1,8 +1,8 @@ # Code Coverage -Tusk Drift can collect code coverage during test replay, showing which lines of your service code each trace test exercises. This helps you understand test value, identify redundant tests, and measure suite completeness. +Tusk Drift can collect code coverage during test replay, showing which lines of your service code each trace test exercises. -Coverage works with Node.js (JavaScript, TypeScript, ESM, CJS) and Python. +Coverage works with Node.js and Python. ## Enabling Coverage @@ -10,14 +10,14 @@ There are two ways to enable coverage: ### Config-driven (for CI) -Add `coverage.enabled: true` to `.tusk/config.yaml`. Coverage is automatically collected during validation runs on the default branch β€” no CI changes needed. +Add `coverage.enabled: true` to `.tusk/config.yaml`. Coverage is automatically collected during validation runs on the default branch. No CI changes needed. ```yaml coverage: enabled: true ``` -Config-driven coverage is **silent** β€” no console output. Data is collected for backend upload during suite validation. +Config-driven coverage is silent (no console output). Data is collected for backend upload during suite validation. ### Flag-driven (for local dev) @@ -126,10 +126,10 @@ In TUI mode, the aggregate summary appears in the service logs panel after all t ### LCOV export ```bash -tusk drift run --show-coverage --coverage-output coverage.lcov +tusk drift run --cloud --show-coverage --coverage-output coverage.lcov --print ``` -Standard LCOV format, compatible with Codecov, Coveralls, SonarQube, VS Code coverage gutters, JetBrains, and most other tools. +Compatible with Codecov, Coveralls, SonarQube, VS Code, and most coverage tools. **Note on validation runs:** - **In-suite tests** are always included in coverage output, even if they fail (a failing test still exercises code paths). @@ -139,7 +139,7 @@ Standard LCOV format, compatible with Codecov, Coveralls, SonarQube, VS Code cov ### JSON export ```bash -tusk drift run --show-coverage --coverage-output coverage.json +tusk drift run --cloud --show-coverage --coverage-output coverage.json --print ``` JSON includes three top-level fields: From 5771f78c3b04272466d21bd35a3ccab234ce03bf Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Fri, 3 Apr 2026 15:27:58 -0700 Subject: [PATCH 23/36] fix: address bugbot review feedback - Filter draft tests from JSON export per-test data (not just aggregate) - Add filterInSuiteRecords to FormatCoverageSummaryLines (TUI consistency) - Call ProcessCoverage in TUI mode so --coverage-output writes file - Suppress double coverage display in TUI (FormatCoverageSummaryLines + ProcessCoverage) - Show "Coverage written to" message in TUI service logs --- internal/runner/coverage.go | 12 +++++++++++- internal/runner/executor.go | 8 ++++++++ internal/tui/test_executor.go | 13 ++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 7893261..c384684 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -478,6 +478,7 @@ func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) []st return nil } + records = filterInSuiteRecords(records) aggregate := mergeWithBaseline(e.coverageBaseline, records) aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) @@ -543,9 +544,18 @@ type CoverageExport struct { func WriteCoverageJSON(path string, aggregate CoverageSnapshot, perTest map[string]map[string]CoverageFileDiff, records []CoverageTestRecord) error { summary := ComputeCoverageSummary(aggregate, perTest, records) - // Filter per-test data to only include files present in the (filtered) aggregate + // Build set of allowed test IDs from the filtered in-suite records + allowedTestIDs := make(map[string]struct{}, len(records)) + for _, r := range records { + allowedTestIDs[r.TestID] = struct{}{} + } + + // Filter per-test data to only include in-suite tests and files present in the (filtered) aggregate filteredPerTest := make(map[string]map[string]CoverageFileDiff, len(perTest)) for testID, testDetail := range perTest { + if _, ok := allowedTestIDs[testID]; !ok { + continue + } filtered := make(map[string]CoverageFileDiff) for fp, fd := range testDetail { if _, ok := aggregate[fp]; ok { diff --git a/internal/runner/executor.go b/internal/runner/executor.go index b46a59f..57a43f3 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -502,6 +502,14 @@ func (e *Executor) SetShowCoverage(show bool) { e.coverageShowOutput = show } +func (e *Executor) IsCoverageShowOutput() bool { + return e.coverageShowOutput +} + +func (e *Executor) GetCoverageOutputPath() string { + return e.coverageOutputPath +} + func (e *Executor) SetCoverageOutputPath(path string) { e.coverageOutputPath = path } diff --git a/internal/tui/test_executor.go b/internal/tui/test_executor.go index 076344e..15a3f48 100644 --- a/internal/tui/test_executor.go +++ b/internal/tui/test_executor.go @@ -750,7 +750,7 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.addServiceLog("\n" + strings.Repeat("=", 60)) m.addServiceLog("🏁 All tests completed!") - // Show aggregate coverage summary in service logs + // Show aggregate coverage summary in service logs and write output file if m.executor.IsCoverageEnabled() { records := m.executor.GetCoverageRecords() if summaryLines := m.executor.FormatCoverageSummaryLines(records); len(summaryLines) > 0 { @@ -759,6 +759,17 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.addServiceLog(line) } } + // Write coverage output file if requested. Suppress console display + // since we already showed the summary above via FormatCoverageSummaryLines. + savedShowOutput := m.executor.IsCoverageShowOutput() + m.executor.SetShowCoverage(false) + if err := m.executor.ProcessCoverage(records); err != nil { + m.addServiceLog(fmt.Sprintf("⚠️ Failed to process coverage: %v", err)) + } + m.executor.SetShowCoverage(savedShowOutput) + if outputPath := m.executor.GetCoverageOutputPath(); outputPath != "" { + m.addServiceLog(fmt.Sprintf("πŸ“„ Coverage written to %s", outputPath)) + } } // All-tests completed upload (non-blocking) From 2ae28c7b3019192dfa5c7aae6547adb75e534afc Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 14:53:32 -0700 Subject: [PATCH 24/36] feat: coverage backend upload integration - Upload per-test coverage via TraceTestCoverageData on TraceTestResult - Upload baseline via CoverageBaseline on UpdateDriftRunCIStatusRequest - Send startup-covered lines for consistent aggregate computation - Callback reorder: snapshot before upload so data is available - GetCoverageBaselineForUpload merges baseline + per-test for full denominator - Use LineRange proto for compact range representation - Increase coverage snapshot timeout to 60s - Baseline counts set to original values (startup lines count as covered) --- cmd/run.go | 40 +++++++------- internal/runner/coverage.go | 22 ++------ internal/runner/executor.go | 23 ++++++++ internal/runner/results_upload.go | 90 +++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 35 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index dcfa9a7..a21c33e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -398,9 +398,7 @@ func runTests(cmd *cobra.Command, args []string) error { res, test, ) - // TODO-COVERAGE-TRACKING: Include per-test coverage in the upload. - // Add TraceTestCoverageData to UploadSingleTestResult (or piggyback on existing proto). - // Data source: executor.GetTestCoverageDetail(test.TraceID) + mu.Lock() attemptedCount++ @@ -433,16 +431,13 @@ func runTests(cmd *cobra.Command, args []string) error { }) } - // Coverage: wrap the OnTestCompleted callback to take snapshots between tests + // Coverage: wrap the OnTestCompleted callback to take snapshots between tests. + // Snapshot runs BEFORE the existing callback (which uploads results) so that + // per-test coverage data is available when building the upload proto. if coverageEnabled { existingCallback := executor.OnTestCompleted executor.SetOnTestCompleted(func(res runner.TestResult, test runner.Test) { - if existingCallback != nil { - existingCallback(res, test) - } - - // Take coverage snapshot. The SDK resets counters on each call, - // so the response contains ONLY coverage from this test. + // Take coverage snapshot FIRST so data is available for upload lineCounts, err := executor.TakeCoverageSnapshot() if err != nil { log.Warn("Failed to take coverage snapshot", "testID", test.TraceID, "error", err) @@ -470,6 +465,12 @@ func runTests(cmd *cobra.Command, args []string) error { log.UserProgress(fmt.Sprintf(" ↳ coverage: %d lines across %d files", totalLines, len(detail))) } } + + // Now run the existing callback (which uploads results). + // Coverage data is available via GetTestCoverageDetail() for the upload. + if existingCallback != nil { + existingCallback(res, test) + } }) } @@ -791,7 +792,11 @@ func runTests(cmd *cobra.Command, args []string) error { passed, failed := countPassedFailed(results) statusMessage = fmt.Sprintf("Validation complete: %d passed, %d failed", passed, failed) } - if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, statusMessage); err != nil { + var interactiveCoverageBaseline runner.CoverageSnapshot + if coverageEnabled && isValidation { + interactiveCoverageBaseline = executor.GetCoverageBaselineForUpload() + } + if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, interactiveCoverageBaseline, commitSha, statusMessage); err != nil { log.Warn("Interactive: cloud finalize failed", "error", err) } mu.Lock() @@ -978,18 +983,17 @@ func runTests(cmd *cobra.Command, args []string) error { } // streamed is always true here so this only updates the CI status // Does NOT upload results to the backend as they are already uploaded via UploadSingleTestResult during the callback - if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, statusMessage); err != nil { + // Coverage baseline (if enabled) is piggybacked on this status update + var headlessCoverageBaseline runner.CoverageSnapshot + if coverageEnabled && isValidation { + headlessCoverageBaseline = executor.GetCoverageBaselineForUpload() + } + if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, headlessCoverageBaseline, commitSha, statusMessage); err != nil { log.Warn("Headless: cloud finalize failed", "error", err) } if isValidation { log.Println("\nSuite validation completed - backend will process results and update suite") } - // TODO-COVERAGE-TRACKING: Upload coverage baseline after validation run completes. - // When coverageEnabled && isValidation, upload the baseline snapshot to backend: - // - endpoint: UploadCoverageBaseline (new) - // - data: executor.coverageBaseline (all coverable lines + branch data) - // - identifiers: driftRunID, observable_service_id, commit_sha - // This provides the denominator for coverage % calculations. mu.Lock() log.Stderr(fmt.Sprintf("\nSuccessfully uploaded %d/%d test results", uploadedCount, attemptedCount)) if attemptedCount > uploadedCount && lastUploadErr != nil { diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index c384684..46f23ea 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -18,7 +18,7 @@ import ( const ( coverageBaselineMaxRetries = 15 coverageBaselineRetryDelay = 200 * time.Millisecond - coverageSnapshotTimeout = 30 * time.Second + coverageSnapshotTimeout = 60 * time.Second ) // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. @@ -205,21 +205,6 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { } } - // TODO-COVERAGE-TRACKING: Upload coverage data to backend during validation runs. - // Upload ALL records (including drafts) β€” backend needs draft coverage for promotion. - // The backend computes the in-suite aggregate itself after promotion decisions. - // When running with --ci and isValidation=true, upload: - // 1. Coverage baseline (all coverable lines) β€” one per validation run - // 2. Per-test coverage records β€” one per trace test (both IN_SUITE and DRAFT) - // Backend uses this for: - // - Coverage-aware test promotion (which drafts add unique coverage?) - // - Coverage history tracking (coverage_snapshot table, IN_SUITE only) - // - Dashboard display (per-file, per-test attribution) - // Backend computes the final aggregate from IN_SUITE tests ONLY (post-promotion). - // The per-test data for rejected drafts is stored but excluded from the aggregate. - // Upload should use existing UploadTraceTestResults piggybacking (add TraceTestCoverageData) - // and a new UploadCoverageBaseline endpoint. - return nil } @@ -232,7 +217,10 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) CoverageSnapshot { merged := make(CoverageSnapshot) - // Deep-copy baseline (don't mutate the original) + // Deep-copy baseline (don't mutate the original). + // Baseline lines include startup-covered counts (count > 0 for lines executed + // during module loading). These count toward "covered" in the aggregate, + // matching industry standard behavior (Istanbul, NYC, coverage.py, etc.). if baseline != nil { for filePath, fileData := range baseline { lines := make(map[string]int, len(fileData.Lines)) diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 57a43f3..4fcd294 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -510,6 +510,29 @@ func (e *Executor) GetCoverageOutputPath() string { return e.coverageOutputPath } +// GetCoverageBaselineForUpload computes the full baseline by merging the raw baseline +// with all per-test records. This ensures the denominator includes lines discovered +// during test execution that weren't in the initial baseline snapshot. +func (e *Executor) GetCoverageBaselineForUpload() CoverageSnapshot { + e.coverageBaselineMu.Lock() + baseline := e.coverageBaseline + e.coverageBaselineMu.Unlock() + + records := e.GetCoverageRecords() + if baseline == nil && len(records) == 0 { + return nil + } + + // Merge baseline with ALL per-test records (not filtered by suite status) + // to get the complete set of coverable lines for the denominator + aggregate := mergeWithBaseline(baseline, records) + + // Apply include/exclude patterns + aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + + return aggregate +} + func (e *Executor) SetCoverageOutputPath(path string) { e.coverageOutputPath = path } diff --git a/internal/runner/results_upload.go b/internal/runner/results_upload.go index 35acd68..4e5ffa4 100644 --- a/internal/runner/results_upload.go +++ b/internal/runner/results_upload.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "os" + "sort" + "strconv" "strings" "time" @@ -110,6 +112,8 @@ func ReportDriftRunSuccess( driftRunID string, authOptions api.AuthOptions, results []TestResult, + coverageBaseline CoverageSnapshot, + commitSha string, statusMessageOverride ...string, ) error { // Note: We always report SUCCESS status here unless there was an error executing tests. @@ -125,9 +129,69 @@ func ReportDriftRunSuccess( CiStatus: finalStatus, CiStatusMessage: &statusMessage, } + + // Attach coverage baseline if available + if coverageBaseline != nil { + statusReq.CoverageBaseline = buildCoverageBaselineProto(coverageBaseline, commitSha) + } + return client.UpdateDriftRunCIStatus(ctx, statusReq, authOptions) } +func buildCoverageBaselineProto(snapshot CoverageSnapshot, commitSha string) *backend.CoverageBaseline { + baseline := &backend.CoverageBaseline{ + CommitSha: commitSha, + CoverableLinesByFile: make(map[string]*backend.FileLineRanges), + StartupCoveredLinesByFile: make(map[string]*backend.FileLineRanges), + } + totalCoverable := int32(0) + for filePath, fileData := range snapshot { + totalCoverable += int32(len(fileData.Lines)) + + var allLines []int32 + var coveredLines []int32 + for lineStr, count := range fileData.Lines { + if n, err := strconv.Atoi(lineStr); err == nil { + allLines = append(allLines, int32(n)) + if count > 0 { + coveredLines = append(coveredLines, int32(n)) + } + } + } + + sort.Slice(allLines, func(i, j int) bool { return allLines[i] < allLines[j] }) + baseline.CoverableLinesByFile[filePath] = toLineRangesProto(allLines) + + if len(coveredLines) > 0 { + sort.Slice(coveredLines, func(i, j int) bool { return coveredLines[i] < coveredLines[j] }) + baseline.StartupCoveredLinesByFile[filePath] = toLineRangesProto(coveredLines) + } + } + baseline.TotalCoverableLines = totalCoverable + return baseline +} + +// toLineRangesProto compresses sorted int32s into LineRange protos. +// [1,2,3,5,6,10] -> [{1,3},{5,6},{10,10}] +func toLineRangesProto(sorted []int32) *backend.FileLineRanges { + if len(sorted) == 0 { + return &backend.FileLineRanges{} + } + var ranges []*backend.LineRange + start, end := sorted[0], sorted[0] + for i := 1; i < len(sorted); i++ { + if sorted[i] == end+1 { + end = sorted[i] + } else { + ranges = append(ranges, &backend.LineRange{Start: start, End: end}) + start = sorted[i] + end = sorted[i] + } + } + ranges = append(ranges, &backend.LineRange{Start: start, End: end}) + return &backend.FileLineRanges{Ranges: ranges} +} + func BuildTraceTestResultsProto(e *Executor, results []TestResult, tests []Test) []*backend.TraceTestResult { out := make([]*backend.TraceTestResult, 0, len(results)) @@ -251,7 +315,33 @@ func BuildTraceTestResultsProto(e *Executor, results []TestResult, tests []Test) } } + // Per-test coverage data (if coverage is enabled) + if e != nil && e.IsCoverageEnabled() { + detail := e.GetTestCoverageDetail(r.TestID) + if len(detail) > 0 { + covData := &backend.TraceTestCoverageData{ + CoveredLinesByFile: make(map[string]*backend.FileLineRanges), + } + totalCovered := int32(0) + for filePath, fd := range detail { + totalCovered += int32(fd.CoveredCount) + sorted := toInt32Slice(fd.CoveredLines) + covData.CoveredLinesByFile[filePath] = toLineRangesProto(sorted) + } + covData.TotalCoveredLines = totalCovered + tr.CoverageData = covData + } + } + out = append(out, tr) } return out } + +func toInt32Slice(ints []int) []int32 { + result := make([]int32, len(ints)) + for i, v := range ints { + result[i] = int32(v) + } + return result +} From 73ae6a2e390c169bdf6602fb9cfcb42f61ab654d Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 17:08:04 -0700 Subject: [PATCH 25/36] chore: update tusk-drift-schemas to v0.1.34, remove local replace --- go.mod | 6 ++---- go.sum | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a3f9502..2323e50 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.25.0 require ( github.com/Use-Tusk/fence v0.1.36 - github.com/Use-Tusk/tusk-drift-schemas v0.1.32 + github.com/Use-Tusk/tusk-drift-schemas v0.1.34 github.com/agnivade/levenshtein v1.0.3 github.com/aymanbagabas/go-osc52/v2 v2.0.1 + github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 github.com/charmbracelet/bubbletea v1.3.9 github.com/charmbracelet/glamour v0.10.0 @@ -36,7 +37,6 @@ require ( github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/catppuccin/go v0.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect @@ -74,5 +74,3 @@ require ( golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.29.0 // indirect ) - -replace github.com/Use-Tusk/tusk-drift-schemas => /Users/sohil/projects/tusk-drift-schemas diff --git a/go.sum b/go.sum index 97730e8..7670e7e 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,8 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Use-Tusk/fence v0.1.36 h1:8S15y8cp3X+xXukx6AN0Ky/aX9/dZyW3fLw5XOQ8YtE= github.com/Use-Tusk/fence v0.1.36/go.mod h1:YkowBDzXioVKJE16vg9z3gSVC6vhzkIZZw2dFf7MW/o= -github.com/Use-Tusk/tusk-drift-schemas v0.1.30 h1:A45pJ/Za6BLIfTLF53BhuzKHHSJ9L7dXEisnuKT5dTc= -github.com/Use-Tusk/tusk-drift-schemas v0.1.30/go.mod h1:pa3EvTj9kKxl9f904RVFkj9YK1zB75QogboKi70zalM= -github.com/Use-Tusk/tusk-drift-schemas v0.1.32 h1:9+q1RH0036rG3RDjVEeUf0ejMsVP7AxqJ8uQ+XPPCH8= -github.com/Use-Tusk/tusk-drift-schemas v0.1.32/go.mod h1:pa3EvTj9kKxl9f904RVFkj9YK1zB75QogboKi70zalM= +github.com/Use-Tusk/tusk-drift-schemas v0.1.34 h1:OUXsA4sfBMA/HCuPqYdfl5EP9+Jq+hYenAmw4wwrEVo= +github.com/Use-Tusk/tusk-drift-schemas v0.1.34/go.mod h1:pa3EvTj9kKxl9f904RVFkj9YK1zB75QogboKi70zalM= github.com/agnivade/levenshtein v1.0.3 h1:M5ZnqLOoZR8ygVq0FfkXsNOKzMCk0xRiow0R5+5VkQ0= github.com/agnivade/levenshtein v1.0.3/go.mod h1:4SFRZbbXWLF4MU1T9Qg0pGgH3Pjs+t6ie5efyrwRJXs= github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= From a871e6f7b1f4ee3705c3b384a10275df87121abe Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 17:33:54 -0700 Subject: [PATCH 26/36] fix: address lint errors, integer overflow warnings, and coverage callback safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix file permissions (0755β†’0750, 0644β†’0600) for gosec - Add nolint:gosec for safe intβ†’int32 conversions (line numbers/counts) - Remove unnecessary nil check around range (gosimple) - Check os.RemoveAll return value (errcheck) - Run gofumpt for formatting - Ensure existingCallback runs even when TakeCoverageSnapshot fails --- cmd/run.go | 45 ++++++++++---------- internal/runner/coverage.go | 71 +++++++++++++++---------------- internal/runner/results_upload.go | 12 +++--- internal/runner/service.go | 2 +- 4 files changed, 64 insertions(+), 66 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 81ab749..32dc73f 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -463,7 +463,6 @@ func runTests(cmd *cobra.Command, args []string) error { test, ) - mu.Lock() attemptedCount++ if err != nil { @@ -502,32 +501,34 @@ func runTests(cmd *cobra.Command, args []string) error { if coverageEnabled { existingCallback := executor.OnTestCompleted executor.SetOnTestCompleted(func(res runner.TestResult, test runner.Test) { - // Take coverage snapshot FIRST so data is available for upload + // Take coverage snapshot FIRST so data is available for upload. + // Always continue to existingCallback even on error so test results still upload. lineCounts, err := executor.TakeCoverageSnapshot() if err != nil { log.Warn("Failed to take coverage snapshot", "testID", test.TraceID, "error", err) - return } - executor.AddCoverageRecord(runner.CoverageTestRecord{ - TestID: test.TraceID, - TestName: test.DisplayName, - SuiteStatus: test.SuiteStatus, - Coverage: lineCounts, - }) - - // Store detail for TUI display - detail := runner.SnapshotToCoverageDetail(lineCounts) - executor.SetTestCoverageDetail(test.TraceID, detail) - - // Print sub-line in --print mode when --show-coverage is active - if !interactive && showCoverage { - totalLines := 0 - for _, fd := range detail { - totalLines += fd.CoveredCount - } - if totalLines > 0 { - log.UserProgress(fmt.Sprintf(" ↳ coverage: %d lines across %d files", totalLines, len(detail))) + if err == nil { + executor.AddCoverageRecord(runner.CoverageTestRecord{ + TestID: test.TraceID, + TestName: test.DisplayName, + SuiteStatus: test.SuiteStatus, + Coverage: lineCounts, + }) + + // Store detail for TUI display + detail := runner.SnapshotToCoverageDetail(lineCounts) + executor.SetTestCoverageDetail(test.TraceID, detail) + + // Print sub-line in --print mode when --show-coverage is active + if !interactive && showCoverage { + totalLines := 0 + for _, fd := range detail { + totalLines += fd.CoveredCount + } + if totalLines > 0 { + log.UserProgress(fmt.Sprintf(" ↳ coverage: %d lines across %d files", totalLines, len(detail))) + } } } diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 46f23ea..28d87e5 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -16,9 +16,9 @@ import ( ) const ( - coverageBaselineMaxRetries = 15 - coverageBaselineRetryDelay = 200 * time.Millisecond - coverageSnapshotTimeout = 60 * time.Second + coverageBaselineMaxRetries = 15 + coverageBaselineRetryDelay = 200 * time.Millisecond + coverageSnapshotTimeout = 60 * time.Second ) // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. @@ -107,12 +107,12 @@ type CoverageTestRecord struct { // CoverageFileDiff represents per-test coverage for a single file. type CoverageFileDiff struct { - CoveredLines []int `json:"covered_lines"` - CoverableLines int `json:"coverable_lines"` - CoveredCount int `json:"covered_count"` - TotalBranches int `json:"total_branches"` - CoveredBranches int `json:"covered_branches"` - Branches map[string]BranchInfo `json:"branches,omitempty"` + CoveredLines []int `json:"covered_lines"` + CoverableLines int `json:"coverable_lines"` + CoveredCount int `json:"covered_count"` + TotalBranches int `json:"total_branches"` + CoveredBranches int `json:"covered_branches"` + Branches map[string]BranchInfo `json:"branches,omitempty"` } // SnapshotToCoverageDetail converts a CoverageSnapshot to per-file CoverageFileDiff format. @@ -187,7 +187,7 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { outPath = filepath.Join(cwd, outPath) } } - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(outPath), 0o750); err != nil { return fmt.Errorf("failed to create coverage output directory: %w", err) } @@ -221,22 +221,20 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) // Baseline lines include startup-covered counts (count > 0 for lines executed // during module loading). These count toward "covered" in the aggregate, // matching industry standard behavior (Istanbul, NYC, coverage.py, etc.). - if baseline != nil { - for filePath, fileData := range baseline { - lines := make(map[string]int, len(fileData.Lines)) - for line, count := range fileData.Lines { - lines[line] = count - } - branches := make(map[string]BranchInfo, len(fileData.Branches)) - for line, info := range fileData.Branches { - branches[line] = info // BranchInfo is a value type, safe to copy - } - merged[filePath] = FileCoverageData{ - Lines: lines, - TotalBranches: fileData.TotalBranches, - CoveredBranches: fileData.CoveredBranches, - Branches: branches, - } + for filePath, fileData := range baseline { + lines := make(map[string]int, len(fileData.Lines)) + for line, count := range fileData.Lines { + lines[line] = count + } + branches := make(map[string]BranchInfo, len(fileData.Branches)) + for line, info := range fileData.Branches { + branches[line] = info // BranchInfo is a value type, safe to copy + } + merged[filePath] = FileCoverageData{ + Lines: lines, + TotalBranches: fileData.TotalBranches, + CoveredBranches: fileData.CoveredBranches, + Branches: branches, } } @@ -296,14 +294,14 @@ type CoverageSummary struct { } type CoverageAggregate struct { - TotalCoverableLines int `json:"total_coverable_lines"` - TotalCoveredLines int `json:"total_covered_lines"` - CoveragePct float64 `json:"coverage_pct"` - TotalFiles int `json:"total_files"` - CoveredFiles int `json:"covered_files"` - TotalBranches int `json:"total_branches"` - CoveredBranches int `json:"covered_branches"` - BranchCoveragePct float64 `json:"branch_coverage_pct"` + TotalCoverableLines int `json:"total_coverable_lines"` + TotalCoveredLines int `json:"total_covered_lines"` + CoveragePct float64 `json:"coverage_pct"` + TotalFiles int `json:"total_files"` + CoveredFiles int `json:"covered_files"` + TotalBranches int `json:"total_branches"` + CoveredBranches int `json:"covered_branches"` + BranchCoveragePct float64 `json:"branch_coverage_pct"` } type CoverageFileSummary struct { @@ -565,7 +563,7 @@ func WriteCoverageJSON(path string, aggregate CoverageSnapshot, perTest map[stri if err != nil { return err } - return os.WriteFile(path, data, 0644) + return os.WriteFile(path, data, 0o600) } // WriteCoverageLCOV writes aggregate coverage data in LCOV format. @@ -640,7 +638,7 @@ func WriteCoverageLCOV(path string, aggregate CoverageSnapshot) error { b.WriteString("end_of_record\n") } - return os.WriteFile(path, []byte(b.String()), 0644) + return os.WriteFile(path, []byte(b.String()), 0o600) } // filterInSuiteRecords returns only records from in-suite tests. @@ -681,7 +679,6 @@ func dedup(sorted []int) []int { return result } - // normalizeCoveragePaths converts absolute file paths to repo-relative paths. // // For non-Docker: uses git root as the base (handles monorepos, cd into subdirs). diff --git a/internal/runner/results_upload.go b/internal/runner/results_upload.go index ba03a8a..3007aea 100644 --- a/internal/runner/results_upload.go +++ b/internal/runner/results_upload.go @@ -139,17 +139,17 @@ func buildCoverageBaselineProto(snapshot CoverageSnapshot, commitSha string) *ba CoverableLinesByFile: make(map[string]*backend.FileLineRanges), StartupCoveredLinesByFile: make(map[string]*backend.FileLineRanges), } - totalCoverable := int32(0) + totalCoverable := int32(0) //nolint:gosec // line counts are safely within int32 range for filePath, fileData := range snapshot { - totalCoverable += int32(len(fileData.Lines)) + totalCoverable += int32(len(fileData.Lines)) //nolint:gosec // line counts are safely within int32 range var allLines []int32 var coveredLines []int32 for lineStr, count := range fileData.Lines { if n, err := strconv.Atoi(lineStr); err == nil { - allLines = append(allLines, int32(n)) + allLines = append(allLines, int32(n)) //nolint:gosec // line numbers are safely within int32 range if count > 0 { - coveredLines = append(coveredLines, int32(n)) + coveredLines = append(coveredLines, int32(n)) //nolint:gosec // line numbers are safely within int32 range } } } @@ -319,7 +319,7 @@ func BuildTraceTestResultsProto(e *Executor, results []TestResult, tests []Test) } totalCovered := int32(0) for filePath, fd := range detail { - totalCovered += int32(fd.CoveredCount) + totalCovered += int32(fd.CoveredCount) //nolint:gosec // coverage counts are safely within int32 range sorted := toInt32Slice(fd.CoveredLines) covData.CoveredLinesByFile[filePath] = toLineRangesProto(sorted) } @@ -336,7 +336,7 @@ func BuildTraceTestResultsProto(e *Executor, results []TestResult, tests []Test) func toInt32Slice(ints []int) []int32 { result := make([]int32, len(ints)) for i, v := range ints { - result[i] = int32(v) + result[i] = int32(v) //nolint:gosec // line numbers are safely within int32 range } return result } diff --git a/internal/runner/service.go b/internal/runner/service.go index 7d02859..e65d6f1 100644 --- a/internal/runner/service.go +++ b/internal/runner/service.go @@ -336,7 +336,7 @@ func (e *Executor) StopService() error { } // Clean up V8 coverage temp directory if e.coverageTempDir != "" { - os.RemoveAll(e.coverageTempDir) + _ = os.RemoveAll(e.coverageTempDir) e.coverageTempDir = "" } log.ServiceLog("Service stopped") From 6c3438381c4f5d691c80f513cef6f6fa16b95a27 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 18:04:48 -0700 Subject: [PATCH 27/36] style: fix remaining gofumpt formatting issues --- internal/runner/coverage_test.go | 1 - internal/runner/executor.go | 12 ++++++------ internal/runner/results_upload.go | 2 +- internal/runner/types.go | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go index 110983b..0e86edb 100644 --- a/internal/runner/coverage_test.go +++ b/internal/runner/coverage_test.go @@ -336,7 +336,6 @@ func TestNormalizeCoveragePaths(t *testing.T) { // We test the function handles edge cases; full integration is tested E2E. } - func TestMatchGlob(t *testing.T) { tests := []struct { path string diff --git a/internal/runner/executor.go b/internal/runner/executor.go index d19d0db..2933f10 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -101,12 +101,12 @@ type Executor struct { coverageIncludePatterns []string coverageExcludePatterns []string coverageStripPrefix string - coveragePerTest map[string]map[string]CoverageFileDiff - coveragePerTestMu sync.Mutex - coverageBaseline CoverageSnapshot - coverageBaselineMu sync.Mutex - coverageRecords []CoverageTestRecord - coverageRecordsMu sync.Mutex + coveragePerTest map[string]map[string]CoverageFileDiff + coveragePerTestMu sync.Mutex + coverageBaseline CoverageSnapshot + coverageBaselineMu sync.Mutex + coverageRecords []CoverageTestRecord + coverageRecordsMu sync.Mutex } func NewExecutor() *Executor { diff --git a/internal/runner/results_upload.go b/internal/runner/results_upload.go index 3007aea..128308a 100644 --- a/internal/runner/results_upload.go +++ b/internal/runner/results_upload.go @@ -135,7 +135,7 @@ func ReportDriftRunSuccess( func buildCoverageBaselineProto(snapshot CoverageSnapshot, commitSha string) *backend.CoverageBaseline { baseline := &backend.CoverageBaseline{ - CommitSha: commitSha, + CommitSha: commitSha, CoverableLinesByFile: make(map[string]*backend.FileLineRanges), StartupCoveredLinesByFile: make(map[string]*backend.FileLineRanges), } diff --git a/internal/runner/types.go b/internal/runner/types.go index b67342b..93c2dd4 100644 --- a/internal/runner/types.go +++ b/internal/runner/types.go @@ -44,7 +44,6 @@ type TestResult struct { Duration int `json:"duration"` // In milliseconds Deviations []Deviation `json:"deviations,omitempty"` Error string `json:"error,omitempty"` - } type Trace struct { From c704076d413ecb1074c206df4101fe7a504243df Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 18:10:00 -0700 Subject: [PATCH 28/36] fix: add bounds check for strconv.Atoi to int32 conversion (CodeQL) --- internal/runner/results_upload.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/runner/results_upload.go b/internal/runner/results_upload.go index 128308a..04a06e0 100644 --- a/internal/runner/results_upload.go +++ b/internal/runner/results_upload.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "os" "sort" "strconv" @@ -146,10 +147,10 @@ func buildCoverageBaselineProto(snapshot CoverageSnapshot, commitSha string) *ba var allLines []int32 var coveredLines []int32 for lineStr, count := range fileData.Lines { - if n, err := strconv.Atoi(lineStr); err == nil { - allLines = append(allLines, int32(n)) //nolint:gosec // line numbers are safely within int32 range + if n, err := strconv.Atoi(lineStr); err == nil && n >= 0 && n <= math.MaxInt32 { + allLines = append(allLines, int32(n)) //nolint:gosec // bounds checked above if count > 0 { - coveredLines = append(coveredLines, int32(n)) //nolint:gosec // line numbers are safely within int32 range + coveredLines = append(coveredLines, int32(n)) //nolint:gosec // bounds checked above } } } From 0582823ee707fc70021b7aaf4b394738ce8faa0f Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 18:29:33 -0700 Subject: [PATCH 29/36] fix: branch coverage max instead of sum, populate validation commit SHA, deduplicate aggregate - Branch merging uses max(covered) instead of sum+clamp to prevent inflating coverage when multiple tests cover the same branches - Populate commitSha variable for validation runs so coverage baseline proto gets the correct commit SHA - FormatCoverageSummaryLines returns computed aggregate for reuse by ProcessCoverageWithAggregate, avoiding redundant computation in TUI --- cmd/run.go | 3 ++- internal/runner/coverage.go | 40 ++++++++++++++++++++--------------- internal/tui/test_executor.go | 6 ++++-- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 32dc73f..c2e1ee1 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -247,11 +247,12 @@ func runTests(cmd *cobra.Command, args []string) error { var req *backend.CreateDriftRunRequest if isValidation { + commitSha = getCommitSHAFromEnv() req = &backend.CreateDriftRunRequest{ ObservableServiceId: cfg.Service.ID, CliVersion: version.Version, IsValidationRun: true, - CommitSha: stringPtr(getCommitSHAFromEnv()), + CommitSha: stringPtr(commitSha), BranchName: stringPtr(getBranchFromEnv()), } } else { diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 28d87e5..f402808 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -156,6 +156,12 @@ func SnapshotToCoverageDetail(snapshot CoverageSnapshot) map[string]CoverageFile // All per-test data (including drafts) is retained for backend upload β€” the backend needs draft // coverage for promotion decisions ("does this draft add unique coverage?"). func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { + return e.ProcessCoverageWithAggregate(records, nil) +} + +// ProcessCoverageWithAggregate processes coverage with an optional pre-computed aggregate. +// If aggregate is nil, it will be computed from the records and baseline. +func (e *Executor) ProcessCoverageWithAggregate(records []CoverageTestRecord, precomputed CoverageSnapshot) error { if !e.coverageEnabled || len(records) == 0 { return nil } @@ -164,12 +170,12 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { // (local run, no cloud), include all tests. suiteRecords := filterInSuiteRecords(records) - // Compute aggregate: start with baseline (all coverable lines including count=0), - // then merge in per-test coverage. This gives accurate denominator. - aggregate := mergeWithBaseline(e.coverageBaseline, suiteRecords) - - // Apply include/exclude patterns from config - aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + // Use pre-computed aggregate if provided, otherwise compute it. + aggregate := precomputed + if aggregate == nil { + aggregate = mergeWithBaseline(e.coverageBaseline, suiteRecords) + aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + } // Print summary if --show-coverage was passed (not in silent config-driven mode) if e.coverageShowOutput { @@ -252,7 +258,9 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) for line, count := range fileData.Lines { existing.Lines[line] += count } - // Union branch data: sum covered counts (clamped to total) + // Union branch data: take max of covered counts per branch point. + // Using max (not sum) avoids inflating coverage when multiple tests + // cover the same branches at a given branch point. for line, branchInfo := range fileData.Branches { if existing.Branches == nil { existing.Branches = make(map[string]BranchInfo) @@ -261,11 +269,8 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) if branchInfo.Total > eb.Total { eb.Total = branchInfo.Total } - newCovered := eb.Covered + branchInfo.Covered - if newCovered > eb.Total || newCovered < 0 { // Clamp + overflow guard - eb.Covered = eb.Total - } else { - eb.Covered = newCovered + if branchInfo.Covered > eb.Covered { + eb.Covered = branchInfo.Covered } existing.Branches[line] = eb } @@ -457,18 +462,19 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate } } -// FormatCoverageSummaryLines computes the coverage summary and returns formatted -// lines for the TUI service log panel (aggregate + per-file, no per-test). -func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) []string { +// FormatCoverageSummaryLines computes the aggregate and returns formatted summary lines +// for the TUI service log panel (aggregate + per-file, no per-test). +// Also returns the computed aggregate so callers can reuse it (avoiding redundant computation). +func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) ([]string, CoverageSnapshot) { if !e.coverageEnabled || len(records) == 0 { - return nil + return nil, nil } records = filterInSuiteRecords(records) aggregate := mergeWithBaseline(e.coverageBaseline, records) aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) - return e.formatCoverageSummary(summary) + return e.formatCoverageSummary(summary), aggregate } // filterCoverageByPatterns applies include/exclude glob patterns to a snapshot. diff --git a/internal/tui/test_executor.go b/internal/tui/test_executor.go index 15a3f48..1436f3c 100644 --- a/internal/tui/test_executor.go +++ b/internal/tui/test_executor.go @@ -753,7 +753,8 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Show aggregate coverage summary in service logs and write output file if m.executor.IsCoverageEnabled() { records := m.executor.GetCoverageRecords() - if summaryLines := m.executor.FormatCoverageSummaryLines(records); len(summaryLines) > 0 { + summaryLines, aggregate := m.executor.FormatCoverageSummaryLines(records) + if len(summaryLines) > 0 { m.addServiceLog("") for _, line := range summaryLines { m.addServiceLog(line) @@ -761,9 +762,10 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } // Write coverage output file if requested. Suppress console display // since we already showed the summary above via FormatCoverageSummaryLines. + // Pass pre-computed aggregate to avoid redundant computation. savedShowOutput := m.executor.IsCoverageShowOutput() m.executor.SetShowCoverage(false) - if err := m.executor.ProcessCoverage(records); err != nil { + if err := m.executor.ProcessCoverageWithAggregate(records, aggregate); err != nil { m.addServiceLog(fmt.Sprintf("⚠️ Failed to process coverage: %v", err)) } m.executor.SetShowCoverage(savedShowOutput) From d6ffbd41a102216c1fd09837f6d70facb8514ae4 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 19:13:23 -0700 Subject: [PATCH 30/36] fix: revert branch coverage to sum+clamp, add LCOV/JSON export tests - Revert branch merging from max to sum+clamp (matches Istanbul/NYC approach) - Add WriteCoverageLCOV tests (format validation, empty, sorted files) - Add WriteCoverageJSON tests (structure validation, empty) --- internal/runner/coverage.go | 14 ++-- internal/runner/coverage_test.go | 126 +++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index f402808..9b89509 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -258,9 +258,10 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) for line, count := range fileData.Lines { existing.Lines[line] += count } - // Union branch data: take max of covered counts per branch point. - // Using max (not sum) avoids inflating coverage when multiple tests - // cover the same branches at a given branch point. + // Union branch data: sum covered counts, clamped to total. + // This is the same approach Istanbul/NYC use when merging reports. + // Without per-arm tracking, sum+clamp is the best approximation + // (optimistic when tests overlap on the same branches). for line, branchInfo := range fileData.Branches { if existing.Branches == nil { existing.Branches = make(map[string]BranchInfo) @@ -269,8 +270,11 @@ func mergeWithBaseline(baseline CoverageSnapshot, records []CoverageTestRecord) if branchInfo.Total > eb.Total { eb.Total = branchInfo.Total } - if branchInfo.Covered > eb.Covered { - eb.Covered = branchInfo.Covered + newCovered := eb.Covered + branchInfo.Covered + if newCovered > eb.Total || newCovered < 0 { + eb.Covered = eb.Total + } else { + eb.Covered = newCovered } existing.Branches[line] = eb } diff --git a/internal/runner/coverage_test.go b/internal/runner/coverage_test.go index 0e86edb..33f56a7 100644 --- a/internal/runner/coverage_test.go +++ b/internal/runner/coverage_test.go @@ -1,6 +1,10 @@ package runner import ( + "encoding/json" + "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -424,3 +428,125 @@ func TestFilterCoverageByPatterns(t *testing.T) { assert.Len(t, result, 6) }) } + +func TestWriteCoverageLCOV(t *testing.T) { + t.Run("writes valid LCOV format", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "src/server.js": FileCoverageData{ + Lines: map[string]int{ + "1": 1, "2": 3, "5": 0, "10": 1, + }, + Branches: map[string]BranchInfo{ + "5": {Total: 2, Covered: 1}, + }, + TotalBranches: 2, + CoveredBranches: 1, + }, + } + path := filepath.Join(t.TempDir(), "coverage.lcov") + err := WriteCoverageLCOV(path, aggregate) + require.NoError(t, err) + + data, err := os.ReadFile(path) //nolint:gosec // test file, path from t.TempDir() + require.NoError(t, err) + content := string(data) + + assert.Contains(t, content, "SF:src/server.js") + assert.Contains(t, content, "DA:1,1") + assert.Contains(t, content, "DA:5,0") + assert.Contains(t, content, "LF:4") + assert.Contains(t, content, "LH:3") + assert.Contains(t, content, "BRF:2") + assert.Contains(t, content, "BRH:1") + assert.Contains(t, content, "end_of_record") + }) + + t.Run("empty snapshot writes empty file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "coverage.lcov") + err := WriteCoverageLCOV(path, CoverageSnapshot{}) + require.NoError(t, err) + + data, err := os.ReadFile(path) //nolint:gosec // test file, path from t.TempDir() + require.NoError(t, err) + assert.Empty(t, strings.TrimSpace(string(data))) + }) + + t.Run("multiple files sorted deterministically", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "z/last.js": FileCoverageData{Lines: map[string]int{"1": 1}}, + "a/first.js": FileCoverageData{Lines: map[string]int{"1": 1}}, + } + path := filepath.Join(t.TempDir(), "coverage.lcov") + err := WriteCoverageLCOV(path, aggregate) + require.NoError(t, err) + + data, err := os.ReadFile(path) //nolint:gosec // test file, path from t.TempDir() + require.NoError(t, err) + content := string(data) + + firstIdx := strings.Index(content, "SF:a/first.js") + lastIdx := strings.Index(content, "SF:z/last.js") + assert.True(t, firstIdx < lastIdx, "files should be sorted alphabetically") + }) +} + +func TestWriteCoverageJSON(t *testing.T) { + t.Run("writes valid JSON with expected structure", func(t *testing.T) { + aggregate := CoverageSnapshot{ + "src/server.js": FileCoverageData{ + Lines: map[string]int{ + "1": 1, "2": 3, "5": 0, + }, + TotalBranches: 4, + CoveredBranches: 2, + }, + } + perTest := map[string]map[string]CoverageFileDiff{ + "test-1": { + "src/server.js": {CoveredLines: []int{1, 2}, CoveredCount: 2}, + }, + } + records := []CoverageTestRecord{ + {TestID: "test-1", TestName: "GET /api"}, + } + + path := filepath.Join(t.TempDir(), "coverage.json") + err := WriteCoverageJSON(path, aggregate, perTest, records) + require.NoError(t, err) + + data, err := os.ReadFile(path) //nolint:gosec // test file, path from t.TempDir() + require.NoError(t, err) + + var result map[string]interface{} + err = json.Unmarshal(data, &result) + require.NoError(t, err) + + // Top-level keys: aggregate (raw snapshot), per_test, summary (computed) + assert.Contains(t, result, "aggregate") + assert.Contains(t, result, "per_test") + assert.Contains(t, result, "summary") + + summary := result["summary"].(map[string]interface{}) + assert.Contains(t, summary, "aggregate") + assert.Contains(t, summary, "per_file") + assert.Contains(t, summary, "timestamp") + + agg := summary["aggregate"].(map[string]interface{}) + assert.Equal(t, float64(3), agg["total_coverable_lines"]) + assert.Equal(t, float64(2), agg["total_covered_lines"]) + }) + + t.Run("empty snapshot writes valid JSON", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "coverage.json") + err := WriteCoverageJSON(path, CoverageSnapshot{}, nil, nil) + require.NoError(t, err) + + data, err := os.ReadFile(path) //nolint:gosec // test file, path from t.TempDir() + require.NoError(t, err) + + var result map[string]interface{} + err = json.Unmarshal(data, &result) + require.NoError(t, err) + assert.Contains(t, result, "aggregate") + }) +} From 174bd69ba46d00cc44c83edf9ceae215816bebec Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 19:37:11 -0700 Subject: [PATCH 31/36] fix: separate startup coverage from test-covered lines in baseline upload GetCoverageBaselineForUpload now returns both the merged snapshot (for coverable lines denominator) and the original baseline (for startup coverage). buildCoverageBaselineProto uses each for its proper purpose, so StartupCoveredLinesByFile only contains lines covered during module loading, not lines covered by test execution. --- cmd/run.go | 12 +++++------ internal/runner/executor.go | 19 ++++++++++------ internal/runner/results_upload.go | 36 ++++++++++++++++++++++--------- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index c2e1ee1..4f724ca 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -868,11 +868,11 @@ func runTests(cmd *cobra.Command, args []string) error { passed, failed := countPassedFailed(results) statusMessage = fmt.Sprintf("Validation complete: %d passed, %d failed", passed, failed) } - var interactiveCoverageBaseline runner.CoverageSnapshot + var interactiveCoverageBaseline, interactiveCoverageOriginal runner.CoverageSnapshot if coverageEnabled && isValidation { - interactiveCoverageBaseline = executor.GetCoverageBaselineForUpload() + interactiveCoverageBaseline, interactiveCoverageOriginal = executor.GetCoverageBaselineForUpload() } - if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, interactiveCoverageBaseline, commitSha, statusMessage); err != nil { + if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, interactiveCoverageBaseline, interactiveCoverageOriginal, commitSha, statusMessage); err != nil { log.Warn("Interactive: cloud finalize failed", "error", err) } mu.Lock() @@ -1080,11 +1080,11 @@ func runTests(cmd *cobra.Command, args []string) error { // streamed is always true here so this only updates the CI status // Does NOT upload results to the backend as they are already uploaded via UploadSingleTestResult during the callback // Coverage baseline (if enabled) is piggybacked on this status update - var headlessCoverageBaseline runner.CoverageSnapshot + var headlessCoverageBaseline, headlessCoverageOriginal runner.CoverageSnapshot if coverageEnabled && isValidation { - headlessCoverageBaseline = executor.GetCoverageBaselineForUpload() + headlessCoverageBaseline, headlessCoverageOriginal = executor.GetCoverageBaselineForUpload() } - if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, headlessCoverageBaseline, commitSha, statusMessage); err != nil { + if err := runner.ReportDriftRunSuccess(context.Background(), client, driftRunID, authOptions, results, headlessCoverageBaseline, headlessCoverageOriginal, commitSha, statusMessage); err != nil { log.Warn("Headless: cloud finalize failed", "error", err) } if isValidation { diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 2933f10..2bb305f 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -508,27 +508,32 @@ func (e *Executor) GetCoverageOutputPath() string { return e.coverageOutputPath } -// GetCoverageBaselineForUpload computes the full baseline by merging the raw baseline -// with all per-test records. This ensures the denominator includes lines discovered -// during test execution that weren't in the initial baseline snapshot. -func (e *Executor) GetCoverageBaselineForUpload() CoverageSnapshot { +// GetCoverageBaselineForUpload returns two snapshots: +// - merged: baseline + all per-test records (complete denominator for coverable lines) +// - originalBaseline: raw baseline only (for startup-covered lines attribution) +// +// The merged snapshot ensures the denominator includes lines discovered during test +// execution that weren't in the initial baseline snapshot. The original baseline is +// kept separate so startup coverage is not conflated with test-driven coverage. +func (e *Executor) GetCoverageBaselineForUpload() (merged CoverageSnapshot, originalBaseline CoverageSnapshot) { e.coverageBaselineMu.Lock() baseline := e.coverageBaseline e.coverageBaselineMu.Unlock() records := e.GetCoverageRecords() if baseline == nil && len(records) == 0 { - return nil + return nil, nil } // Merge baseline with ALL per-test records (not filtered by suite status) // to get the complete set of coverable lines for the denominator aggregate := mergeWithBaseline(baseline, records) - // Apply include/exclude patterns + // Apply include/exclude patterns to both aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + filteredBaseline := filterCoverageByPatterns(baseline, e.coverageIncludePatterns, e.coverageExcludePatterns) - return aggregate + return aggregate, filteredBaseline } func (e *Executor) SetCoverageOutputPath(path string) { diff --git a/internal/runner/results_upload.go b/internal/runner/results_upload.go index 04a06e0..a23b48c 100644 --- a/internal/runner/results_upload.go +++ b/internal/runner/results_upload.go @@ -109,6 +109,7 @@ func ReportDriftRunSuccess( authOptions api.AuthOptions, results []TestResult, coverageBaseline CoverageSnapshot, + coverageOriginalBaseline CoverageSnapshot, commitSha string, statusMessageOverride ...string, ) error { @@ -128,42 +129,57 @@ func ReportDriftRunSuccess( // Attach coverage baseline if available if coverageBaseline != nil { - statusReq.CoverageBaseline = buildCoverageBaselineProto(coverageBaseline, commitSha) + statusReq.CoverageBaseline = buildCoverageBaselineProto(coverageBaseline, coverageOriginalBaseline, commitSha) } return client.UpdateDriftRunCIStatus(ctx, statusReq, authOptions) } -func buildCoverageBaselineProto(snapshot CoverageSnapshot, commitSha string) *backend.CoverageBaseline { +// buildCoverageBaselineProto builds the proto from two snapshots: +// - merged: all coverable lines (baseline + per-test, for denominator) +// - originalBaseline: raw baseline only (for startup-covered lines) +// +// This separation ensures StartupCoveredLinesByFile only contains lines covered +// during module loading, not lines covered by test execution. +func buildCoverageBaselineProto(merged CoverageSnapshot, originalBaseline CoverageSnapshot, commitSha string) *backend.CoverageBaseline { baseline := &backend.CoverageBaseline{ CommitSha: commitSha, CoverableLinesByFile: make(map[string]*backend.FileLineRanges), StartupCoveredLinesByFile: make(map[string]*backend.FileLineRanges), } + + // Coverable lines from the merged snapshot (complete denominator) totalCoverable := int32(0) //nolint:gosec // line counts are safely within int32 range - for filePath, fileData := range snapshot { + for filePath, fileData := range merged { totalCoverable += int32(len(fileData.Lines)) //nolint:gosec // line counts are safely within int32 range var allLines []int32 - var coveredLines []int32 - for lineStr, count := range fileData.Lines { + for lineStr := range fileData.Lines { if n, err := strconv.Atoi(lineStr); err == nil && n >= 0 && n <= math.MaxInt32 { allLines = append(allLines, int32(n)) //nolint:gosec // bounds checked above - if count > 0 { - coveredLines = append(coveredLines, int32(n)) //nolint:gosec // bounds checked above - } } } - sort.Slice(allLines, func(i, j int) bool { return allLines[i] < allLines[j] }) baseline.CoverableLinesByFile[filePath] = toLineRangesProto(allLines) + } + baseline.TotalCoverableLines = totalCoverable + // Startup-covered lines from the original baseline only + for filePath, fileData := range originalBaseline { + var coveredLines []int32 + for lineStr, count := range fileData.Lines { + if count > 0 { + if n, err := strconv.Atoi(lineStr); err == nil && n >= 0 && n <= math.MaxInt32 { + coveredLines = append(coveredLines, int32(n)) //nolint:gosec // bounds checked above + } + } + } if len(coveredLines) > 0 { sort.Slice(coveredLines, func(i, j int) bool { return coveredLines[i] < coveredLines[j] }) baseline.StartupCoveredLinesByFile[filePath] = toLineRangesProto(coveredLines) } } - baseline.TotalCoverableLines = totalCoverable + return baseline } From 5821c201abcd100ecad4fef5480f9ac823035b79 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Mon, 6 Apr 2026 19:57:04 -0700 Subject: [PATCH 32/36] =?UTF-8?q?fix:=20address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20mutex=20race,=20response=20race,=20JSON=20summary,?= =?UTF-8?q?=20Windows=20paths,=20TUI=20error=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Protect coveragePerTest reads with mutex via GetCoveragePerTestSnapshot() - Register pending response channel before sending coverage snapshot request - Compute JSON summary from filtered per-test data (after include/exclude) - Normalize backslash paths for Windows glob matching - Only log "Coverage written" on success --- internal/runner/coverage.go | 13 ++++++++----- internal/runner/executor.go | 15 +++++++++++++++ internal/runner/server.go | 21 ++++++++++++++++++--- internal/tui/test_executor.go | 5 ++--- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 9b89509..6b6bf23 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -198,7 +198,7 @@ func (e *Executor) ProcessCoverageWithAggregate(records []CoverageTestRecord, pr } if strings.HasSuffix(strings.ToLower(outPath), ".json") { - if err := WriteCoverageJSON(outPath, aggregate, e.coveragePerTest, suiteRecords); err != nil { + if err := WriteCoverageJSON(outPath, aggregate, e.GetCoveragePerTestSnapshot(), suiteRecords); err != nil { return fmt.Errorf("failed to write coverage JSON: %w", err) } } else { @@ -450,7 +450,7 @@ func (e *Executor) formatCoverageSummary(summary CoverageSummary) []string { // printCoverageSummary computes and prints the coverage summary to stderr. func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) { - summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) + summary := ComputeCoverageSummary(aggregate, e.GetCoveragePerTestSnapshot(), records) for _, line := range e.formatCoverageSummary(summary) { log.Stderrln(line) } @@ -477,7 +477,7 @@ func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) ([]s records = filterInSuiteRecords(records) aggregate := mergeWithBaseline(e.coverageBaseline, records) aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) - summary := ComputeCoverageSummary(aggregate, e.coveragePerTest, records) + summary := ComputeCoverageSummary(aggregate, e.GetCoveragePerTestSnapshot(), records) return e.formatCoverageSummary(summary), aggregate } @@ -512,6 +512,7 @@ func filterCoverageByPatterns(snapshot CoverageSnapshot, include, exclude []stri // matchesAnyPattern checks if a file path matches any of the glob patterns. // Uses doublestar for proper ** support. func matchesAnyPattern(filePath string, patterns []string) bool { + filePath = strings.ReplaceAll(filePath, "\\", "/") for _, pattern := range patterns { if matched, _ := doublestar.Match(pattern, filePath); matched { return true @@ -523,6 +524,7 @@ func matchesAnyPattern(filePath string, patterns []string) bool { // matchGlob matches a path against a glob pattern supporting **. // Exported for testing. func matchGlob(filePath, pattern string) bool { + filePath = strings.ReplaceAll(filePath, "\\", "/") matched, _ := doublestar.Match(pattern, filePath) return matched } @@ -538,8 +540,6 @@ type CoverageExport struct { // WriteCoverageJSON writes aggregate + per-test coverage as JSON. func WriteCoverageJSON(path string, aggregate CoverageSnapshot, perTest map[string]map[string]CoverageFileDiff, records []CoverageTestRecord) error { - summary := ComputeCoverageSummary(aggregate, perTest, records) - // Build set of allowed test IDs from the filtered in-suite records allowedTestIDs := make(map[string]struct{}, len(records)) for _, r := range records { @@ -563,6 +563,9 @@ func WriteCoverageJSON(path string, aggregate CoverageSnapshot, perTest map[stri } } + // Compute summary from the filtered per-test data so it matches the exported data + summary := ComputeCoverageSummary(aggregate, filteredPerTest, records) + export := CoverageExport{ Summary: summary, Aggregate: aggregate, diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 2bb305f..7074fc2 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -623,6 +623,21 @@ func (e *Executor) GetTestCoverageDetail(testID string) map[string]CoverageFileD return copied } +// GetCoveragePerTestSnapshot returns a shallow copy of the entire per-test coverage map. +// The outer map is copied so callers can iterate without holding the mutex. +func (e *Executor) GetCoveragePerTestSnapshot() map[string]map[string]CoverageFileDiff { + e.coveragePerTestMu.Lock() + defer e.coveragePerTestMu.Unlock() + if e.coveragePerTest == nil { + return nil + } + copied := make(map[string]map[string]CoverageFileDiff, len(e.coveragePerTest)) + for k, v := range e.coveragePerTest { + copied[k] = v + } + return copied +} + // AddCoverageRecord stores a per-test coverage record. func (e *Executor) AddCoverageRecord(record CoverageTestRecord) { e.coverageRecordsMu.Lock() diff --git a/internal/runner/server.go b/internal/runner/server.go index 3eb4b24..cc5bde5 100644 --- a/internal/runner/server.go +++ b/internal/runner/server.go @@ -1008,13 +1008,28 @@ func (ms *Server) SendCoverageSnapshot(baseline bool) (*core.CoverageSnapshotRes }, } + // Register the pending response channel BEFORE sending so we don't miss + // a fast SDK reply that arrives before the channel is registered. + respChan := make(chan *core.SDKMessage, 1) + ms.pendingMu.Lock() + ms.pendingRequests[requestID] = respChan + ms.pendingMu.Unlock() + + defer func() { + ms.pendingMu.Lock() + delete(ms.pendingRequests, requestID) + ms.pendingMu.Unlock() + }() + if err := ms.sendProtobufResponse(conn, msg); err != nil { return nil, fmt.Errorf("failed to send coverage snapshot request: %w", err) } - response, err := ms.waitForSDKResponse(requestID, coverageSnapshotTimeout) - if err != nil { - return nil, fmt.Errorf("failed to receive coverage snapshot response: %w", err) + var response *core.SDKMessage + select { + case response = <-respChan: + case <-time.After(coverageSnapshotTimeout): + return nil, fmt.Errorf("failed to receive coverage snapshot response: timeout waiting for SDK response") } coverageResp := response.GetCoverageSnapshotResponse() diff --git a/internal/tui/test_executor.go b/internal/tui/test_executor.go index 1436f3c..62b3730 100644 --- a/internal/tui/test_executor.go +++ b/internal/tui/test_executor.go @@ -767,11 +767,10 @@ func (m *testExecutorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.executor.SetShowCoverage(false) if err := m.executor.ProcessCoverageWithAggregate(records, aggregate); err != nil { m.addServiceLog(fmt.Sprintf("⚠️ Failed to process coverage: %v", err)) - } - m.executor.SetShowCoverage(savedShowOutput) - if outputPath := m.executor.GetCoverageOutputPath(); outputPath != "" { + } else if outputPath := m.executor.GetCoverageOutputPath(); outputPath != "" { m.addServiceLog(fmt.Sprintf("πŸ“„ Coverage written to %s", outputPath)) } + m.executor.SetShowCoverage(savedShowOutput) } // All-tests completed upload (non-blocking) From e8f5560ace6bbba027b182c79659d885be49eb98 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Tue, 7 Apr 2026 13:56:50 -0700 Subject: [PATCH 33/36] fix: baseline retry deadline, mutex on reads, concurrency override warning --- cmd/run.go | 3 +++ internal/runner/coverage.go | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 4f724ca..a600b36 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -348,6 +348,9 @@ func runTests(cmd *cobra.Command, args []string) error { } // Coverage requires serial execution (concurrency=1) because per-test // snapshots rely on the SDK resetting counters between tests. + if cmd.Flags().Changed("concurrency") { + log.Warn("Coverage requires concurrency=1; your --concurrency flag is being overridden") + } executor.SetConcurrency(1) if showCoverage { log.Stderrln("➀ Coverage collection enabled (concurrency forced to 1)") diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 6b6bf23..2d6a344 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -31,8 +31,12 @@ func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { // Returns ALL coverable lines (including uncovered at count=0) for the aggregate denominator. // Retries with backoff since the coverage server may not be ready immediately after service start. func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { + deadline := time.Now().Add(30 * time.Second) var lastErr error for attempt := 0; attempt < coverageBaselineMaxRetries; attempt++ { + if time.Now().After(deadline) { + break + } result, err := e.callCoverageEndpoint(true) if err == nil { return result, nil @@ -173,7 +177,10 @@ func (e *Executor) ProcessCoverageWithAggregate(records []CoverageTestRecord, pr // Use pre-computed aggregate if provided, otherwise compute it. aggregate := precomputed if aggregate == nil { - aggregate = mergeWithBaseline(e.coverageBaseline, suiteRecords) + e.coverageBaselineMu.Lock() + baseline := e.coverageBaseline + e.coverageBaselineMu.Unlock() + aggregate = mergeWithBaseline(baseline, suiteRecords) aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) } @@ -420,7 +427,10 @@ func (e *Executor) formatCoverageSummary(summary CoverageSummary) []string { summary.Aggregate.BranchCoveragePct, summary.Aggregate.CoveredBranches, summary.Aggregate.TotalBranches) } coverageMsg += fmt.Sprintf(" across %d files", summary.Aggregate.TotalFiles) - if e.coverageBaseline == nil { + e.coverageBaselineMu.Lock() + baselineNil := e.coverageBaseline == nil + e.coverageBaselineMu.Unlock() + if baselineNil { coverageMsg += " ⚠️ baseline failed - denominator may be incomplete" } lines = append(lines, coverageMsg) @@ -475,7 +485,10 @@ func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) ([]s } records = filterInSuiteRecords(records) - aggregate := mergeWithBaseline(e.coverageBaseline, records) + e.coverageBaselineMu.Lock() + baseline := e.coverageBaseline + e.coverageBaselineMu.Unlock() + aggregate := mergeWithBaseline(baseline, records) aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) summary := ComputeCoverageSummary(aggregate, e.GetCoveragePerTestSnapshot(), records) return e.formatCoverageSummary(summary), aggregate From eba00acca782baa14c47155b2a62afbe9a76f6bc Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Tue, 7 Apr 2026 13:57:28 -0700 Subject: [PATCH 34/36] remove concurrency override warning --- cmd/run.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index a600b36..4f724ca 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -348,9 +348,6 @@ func runTests(cmd *cobra.Command, args []string) error { } // Coverage requires serial execution (concurrency=1) because per-test // snapshots rely on the SDK resetting counters between tests. - if cmd.Flags().Changed("concurrency") { - log.Warn("Coverage requires concurrency=1; your --concurrency flag is being overridden") - } executor.SetConcurrency(1) if showCoverage { log.Stderrln("➀ Coverage collection enabled (concurrency forced to 1)") From 900d4a257bb82042938410798ad47c4c84bcbfc5 Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Tue, 7 Apr 2026 14:00:18 -0700 Subject: [PATCH 35/36] increase baseline retry deadline to 90s for large codebases --- internal/runner/coverage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 2d6a344..0e89e41 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -31,7 +31,7 @@ func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { // Returns ALL coverable lines (including uncovered at count=0) for the aggregate denominator. // Retries with backoff since the coverage server may not be ready immediately after service start. func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { - deadline := time.Now().Add(30 * time.Second) + deadline := time.Now().Add(90 * time.Second) var lastErr error for attempt := 0; attempt < coverageBaselineMaxRetries; attempt++ { if time.Now().After(deadline) { From 843cc85efbf07f6765064007df803bc30fab739c Mon Sep 17 00:00:00 2001 From: Sohil Kshirsagar Date: Tue, 7 Apr 2026 17:32:47 -0700 Subject: [PATCH 36/36] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20context.WithDeadline,=20skip=20aggregate=20without?= =?UTF-8?q?=20baseline,=20CoverageReportView?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use context.WithDeadline for baseline retry instead of manual time check - Reduce max retries from 15 to 4 (baseline succeeds on first attempt in practice) - Skip aggregate upload when baseline is nil to avoid misleading 100% coverage - Add CoverageReportView struct and BuildCoverageReportView builder to centralize the filterInSuiteRecords β†’ mergeWithBaseline β†’ filterByPatterns β†’ ComputeSummary chain --- internal/runner/coverage.go | 97 +++++++++++++++++++++++++------------ internal/runner/executor.go | 9 +++- 2 files changed, 73 insertions(+), 33 deletions(-) diff --git a/internal/runner/coverage.go b/internal/runner/coverage.go index 0e89e41..d04954b 100644 --- a/internal/runner/coverage.go +++ b/internal/runner/coverage.go @@ -1,6 +1,7 @@ package runner import ( + "context" "encoding/json" "fmt" "os" @@ -16,9 +17,10 @@ import ( ) const ( - coverageBaselineMaxRetries = 15 + coverageBaselineMaxRetries = 4 coverageBaselineRetryDelay = 200 * time.Millisecond coverageSnapshotTimeout = 60 * time.Second + coverageBaselineDeadline = 90 * time.Second ) // TakeCoverageSnapshot calls the SDK's coverage snapshot endpoint. @@ -29,12 +31,16 @@ func (e *Executor) TakeCoverageSnapshot() (CoverageSnapshot, error) { // TakeCoverageBaseline calls the SDK's coverage snapshot endpoint with ?baseline=true. // Returns ALL coverable lines (including uncovered at count=0) for the aggregate denominator. -// Retries with backoff since the coverage server may not be ready immediately after service start. +// Retries briefly since the coverage server may not be ready immediately after service start. +// In practice, the SDK initializes coverage before the HTTP server starts, so the baseline +// should succeed on the first attempt. func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { - deadline := time.Now().Add(90 * time.Second) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(coverageBaselineDeadline)) + defer cancel() + var lastErr error for attempt := 0; attempt < coverageBaselineMaxRetries; attempt++ { - if time.Now().After(deadline) { + if ctx.Err() != nil { break } result, err := e.callCoverageEndpoint(true) @@ -42,7 +48,11 @@ func (e *Executor) TakeCoverageBaseline() (CoverageSnapshot, error) { return result, nil } lastErr = err - time.Sleep(coverageBaselineRetryDelay) + + select { + case <-ctx.Done(): + case <-time.After(coverageBaselineRetryDelay): + } } return nil, fmt.Errorf("coverage baseline failed after retries: %w", lastErr) } @@ -155,6 +165,37 @@ func SnapshotToCoverageDetail(snapshot CoverageSnapshot) map[string]CoverageFile return result } +// CoverageReportView is a pre-computed view of coverage data, built once and +// passed to all consumers (print, JSON export, TUI) for consistency. +type CoverageReportView struct { + SuiteRecords []CoverageTestRecord + Aggregate CoverageSnapshot + PerTest map[string]map[string]CoverageFileDiff + Summary CoverageSummary +} + +// BuildCoverageReportView constructs a CoverageReportView by applying suite filtering, +// include/exclude patterns, and computing the summary β€” all exactly once. +func (e *Executor) BuildCoverageReportView(records []CoverageTestRecord) *CoverageReportView { + suiteRecords := filterInSuiteRecords(records) + + e.coverageBaselineMu.Lock() + baseline := e.coverageBaseline + e.coverageBaselineMu.Unlock() + + aggregate := mergeWithBaseline(baseline, suiteRecords) + aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + perTest := e.GetCoveragePerTestSnapshot() + summary := ComputeCoverageSummary(aggregate, perTest, suiteRecords) + + return &CoverageReportView{ + SuiteRecords: suiteRecords, + Aggregate: aggregate, + PerTest: perTest, + Summary: summary, + } +} + // ProcessCoverage computes aggregate coverage, optionally prints summary, writes file, and prepares for upload. // During validation runs, the aggregate and output files only include IN_SUITE tests (not drafts). // All per-test data (including drafts) is retained for backend upload β€” the backend needs draft @@ -163,31 +204,30 @@ func (e *Executor) ProcessCoverage(records []CoverageTestRecord) error { return e.ProcessCoverageWithAggregate(records, nil) } -// ProcessCoverageWithAggregate processes coverage with an optional pre-computed aggregate. -// If aggregate is nil, it will be computed from the records and baseline. +// ProcessCoverageWithAggregate processes coverage with an optional pre-computed view. +// If precomputed is nil, it will be computed from the records and baseline. func (e *Executor) ProcessCoverageWithAggregate(records []CoverageTestRecord, precomputed CoverageSnapshot) error { if !e.coverageEnabled || len(records) == 0 { return nil } - // Filter to in-suite tests for the aggregate. If no suite status is set - // (local run, no cloud), include all tests. - suiteRecords := filterInSuiteRecords(records) - - // Use pre-computed aggregate if provided, otherwise compute it. - aggregate := precomputed - if aggregate == nil { - e.coverageBaselineMu.Lock() - baseline := e.coverageBaseline - e.coverageBaselineMu.Unlock() - aggregate = mergeWithBaseline(baseline, suiteRecords) - aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) + // Use pre-computed aggregate if provided, otherwise build the view. + var aggregate CoverageSnapshot + var suiteRecords []CoverageTestRecord + if precomputed != nil { + aggregate = precomputed + suiteRecords = filterInSuiteRecords(records) + } else { + view := e.BuildCoverageReportView(records) + aggregate = view.Aggregate + suiteRecords = view.SuiteRecords } // Print summary if --show-coverage was passed (not in silent config-driven mode) if e.coverageShowOutput { log.Stderrln("\n➀ Processing coverage data...") - e.printCoverageSummary(suiteRecords, aggregate) + summary := ComputeCoverageSummary(aggregate, e.GetCoveragePerTestSnapshot(), suiteRecords) + e.printCoverageSummary(summary) } // Write coverage file if requested. @@ -458,9 +498,8 @@ func (e *Executor) formatCoverageSummary(summary CoverageSummary) []string { return lines } -// printCoverageSummary computes and prints the coverage summary to stderr. -func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate CoverageSnapshot) { - summary := ComputeCoverageSummary(aggregate, e.GetCoveragePerTestSnapshot(), records) +// printCoverageSummary prints the coverage summary to stderr. +func (e *Executor) printCoverageSummary(summary CoverageSummary) { for _, line := range e.formatCoverageSummary(summary) { log.Stderrln(line) } @@ -476,7 +515,7 @@ func (e *Executor) printCoverageSummary(records []CoverageTestRecord, aggregate } } -// FormatCoverageSummaryLines computes the aggregate and returns formatted summary lines +// FormatCoverageSummaryLines builds a CoverageReportView and returns formatted summary lines // for the TUI service log panel (aggregate + per-file, no per-test). // Also returns the computed aggregate so callers can reuse it (avoiding redundant computation). func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) ([]string, CoverageSnapshot) { @@ -484,14 +523,8 @@ func (e *Executor) FormatCoverageSummaryLines(records []CoverageTestRecord) ([]s return nil, nil } - records = filterInSuiteRecords(records) - e.coverageBaselineMu.Lock() - baseline := e.coverageBaseline - e.coverageBaselineMu.Unlock() - aggregate := mergeWithBaseline(baseline, records) - aggregate = filterCoverageByPatterns(aggregate, e.coverageIncludePatterns, e.coverageExcludePatterns) - summary := ComputeCoverageSummary(aggregate, e.GetCoveragePerTestSnapshot(), records) - return e.formatCoverageSummary(summary), aggregate + view := e.BuildCoverageReportView(records) + return e.formatCoverageSummary(view.Summary), view.Aggregate } // filterCoverageByPatterns applies include/exclude glob patterns to a snapshot. diff --git a/internal/runner/executor.go b/internal/runner/executor.go index 7074fc2..2d2851d 100644 --- a/internal/runner/executor.go +++ b/internal/runner/executor.go @@ -520,8 +520,15 @@ func (e *Executor) GetCoverageBaselineForUpload() (merged CoverageSnapshot, orig baseline := e.coverageBaseline e.coverageBaselineMu.Unlock() + // If no baseline was captured, skip the aggregate upload entirely. + // Without a baseline denominator, coverage % would be near 100% (misleading). + // Per-test coverage is still uploaded via TraceTestCoverageData independently. + if baseline == nil { + return nil, nil + } + records := e.GetCoverageRecords() - if baseline == nil && len(records) == 0 { + if len(records) == 0 { return nil, nil }