Fix core gaps: language context, lenient parsing, chunking, examples, CWE

- Language detection: inject primary language + file counts into prompts
- Lenient JSON: strip comments, fix unescaped newlines, trailing commas;
  regex fallback extracts findings from completely broken JSON
- Chunking: Describe/Improve now truncate large diffs within token budget
- Few-shot examples: each concern has a concrete JSON finding example
- CWE from LLM: added to JSON schema, LLM provides directly, falls back to matcher

Author: Patel230

describe.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/GrayCodeAI/sight/internal/diff"
+	"github.com/GrayCodeAI/sight/internal/review"
 )
 
 // Description is the generated PR summary.
@@ -33,7 +34,7 @@ func Describe(ctx context.Context, rawDiff string, opts ...Option) (*Description
 		return &Description{Title: "No changes", Summary: "Empty diff"}, nil
 	}
 
-	prompt := buildDescribePrompt(files)
+	prompt := buildDescribePrompt(files, cfg.maxTokens)
 
 	resp, err := cfg.provider.Chat(ctx, []Message{
 		{Role: "user", Content: prompt},
@@ -69,7 +70,7 @@ Rules:
 - Risk: consider blast radius, backwards compatibility, data migrations
 - Test plan: concrete steps, not generic "run tests"`
 
-func buildDescribePrompt(files []diff.File) string {
+func buildDescribePrompt(files []diff.File, maxTokens int) string {
 	var b strings.Builder
 	b.WriteString("Generate a PR description for these changes:\n\n")
 	b.WriteString("## Stats\n")
@@ -88,27 +89,50 @@ func buildDescribePrompt(files []diff.File) string {
 		b.WriteString("- " + f.Path + prefix + "\n")
 	}
 
-	b.WriteString("\n## Diff\n\n```diff\n")
+	// Build the diff section, truncating if it would exceed the token budget.
+	// For Describe, keep the file list + stats but truncate the diff to fit.
+	tokenBudget := maxTokens * 3 // leave room for the response
+	headerTokens := review.EstimateTokens(b.String())
+	diffBudget := tokenBudget - headerTokens - 100 // 100 token buffer
+
+	var diffBuilder strings.Builder
+	diffBuilder.WriteString("\n## Diff\n\n```diff\n")
+	truncated := false
+
 	for _, f := range files {
 		if f.Deleted {
 			continue
 		}
-		b.WriteString("--- " + f.Path + "\n")
+		var fileSection strings.Builder
+		fileSection.WriteString("--- " + f.Path + "\n")
 		for _, h := range f.Hunks {
 			for _, l := range h.Lines {
 				switch l.Type {
 				case diff.LineAdded:
-					b.WriteString("+" + l.Content + "\n")
+					fileSection.WriteString("+" + l.Content + "\n")
 				case diff.LineRemoved:
-					b.WriteString("-" + l.Content + "\n")
+					fileSection.WriteString("-" + l.Content + "\n")
 				case diff.LineContext:
-					b.WriteString(" " + l.Content + "\n")
+					fileSection.WriteString(" " + l.Content + "\n")
 				}
 			}
 		}
+
+		sectionTokens := review.EstimateTokens(fileSection.String())
+		currentTokens := review.EstimateTokens(diffBuilder.String())
+		if diffBudget > 0 && currentTokens+sectionTokens > diffBudget {
+			truncated = true
+			break
+		}
+		diffBuilder.WriteString(fileSection.String())
+	}
+
+	diffBuilder.WriteString("```\n")
+	if truncated {
+		diffBuilder.WriteString("\n(diff truncated to fit token budget — file list and stats above are complete)\n")
 	}
-	b.WriteString("```\n")
 
+	b.WriteString(diffBuilder.String())
 	return b.String()
 }
 

improve.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/GrayCodeAI/sight/internal/diff"
+	"github.com/GrayCodeAI/sight/internal/review"
 )
 
 // Improvement represents a suggested code improvement.
@@ -43,6 +44,10 @@ func Improve(ctx context.Context, rawDiff string, opts ...Option) (*ImproveResul
 		return &ImproveResult{}, nil
 	}
 
+	// For Improve, only send the first N files that fit within the token budget.
+	tokenBudget := cfg.maxTokens * 3 // leave room for response
+	files = truncateFilesForBudget(files, tokenBudget)
+
 	prompt := buildImprovePrompt(files)
 
 	resp, err := cfg.provider.Chat(ctx, []Message{
@@ -144,6 +149,52 @@ func parseImprovements(response string) []Improvement {
 	return valid
 }
 
+// truncateFilesForBudget returns a subset of files whose combined estimated
+// token count fits within the given budget. It keeps files in order, including
+// only those that fit.
+func truncateFilesForBudget(files []diff.File, tokenBudget int) []diff.File {
+	if tokenBudget <= 0 {
+		return files
+	}
+
+	overhead := review.EstimateTokens(improveSystemPrompt) + 200
+	available := tokenBudget - overhead
+	if available <= 0 {
+		available = tokenBudget / 2
+	}
+
+	var result []diff.File
+	currentTokens := 0
+	for _, f := range files {
+		if f.Deleted {
+			continue
+		}
+		fileTokens := estimateFileTokensPublic(f)
+		if currentTokens+fileTokens > available && len(result) > 0 {
+			break
+		}
+		result = append(result, f)
+		currentTokens += fileTokens
+	}
+	if len(result) == 0 && len(files) > 0 {
+		// Always include at least the first file.
+		result = append(result, files[0])
+	}
+	return result
+}
+
+// estimateFileTokensPublic approximates the token count for a single file's diff.
+func estimateFileTokensPublic(f diff.File) int {
+	tokens := review.EstimateTokens(f.Path) + 20
+	for _, h := range f.Hunks {
+		tokens += 10
+		for _, l := range h.Lines {
+			tokens += review.EstimateTokens(l.Content) + 1
+		}
+	}
+	return tokens
+}
+
 func extractJSONArray(s string) string {
 	s = strings.TrimSpace(s)
 

internal/review/concerns.go

@@ -28,6 +28,7 @@ type Finding struct {
 	Message   string
 	Fix       string
 	Reasoning string
+	CWE       string
 }
 
 // AllConcerns returns every available concern definition.
@@ -43,7 +44,10 @@ func AllConcerns() []Concern {
 - Missing input validation and sanitization
 - SSRF, open redirects, CORS misconfiguration
 - Race conditions that could be exploited
-- Cryptographic weaknesses`,
+- Cryptographic weaknesses
+
+Example of a correct finding:
+{"file": "handler.go", "line": 42, "severity": "critical", "message": "SQL injection: user input concatenated directly into query string", "fix": "Use parameterized query: db.Query(\"SELECT * FROM users WHERE id = $1\", userID)", "reasoning": "The userID variable comes from r.URL.Query() and is concatenated into the SQL string without sanitization"}`,
 		},
 		{
 			Name: "bugs",
@@ -56,7 +60,10 @@ func AllConcerns() []Concern {
 - Integer overflow/underflow
 - Incorrect boolean logic or operator precedence
 - Unreachable code and dead branches
-- Type assertion failures`,
+- Type assertion failures
+
+Example of a correct finding:
+{"file": "service.go", "line": 87, "severity": "high", "message": "Nil pointer dereference: result used without checking error return from GetUser", "fix": "Add nil check: if result == nil { return fmt.Errorf(\"user not found: %s\", id) }", "reasoning": "GetUser returns (nil, nil) when user is not found, so result.Name on line 88 will panic"}`,
 		},
 		{
 			Name: "performance",
@@ -68,7 +75,10 @@ func AllConcerns() []Concern {
 - Blocking operations that could be concurrent
 - N+1 query patterns
 - Excessive string concatenation without builders
-- Missing connection/resource pooling`,
+- Missing connection/resource pooling
+
+Example of a correct finding:
+{"file": "process.go", "line": 55, "severity": "medium", "message": "Allocation in hot loop: fmt.Sprintf called on every iteration to build key", "fix": "Pre-allocate a strings.Builder outside the loop or use string concatenation for simple key construction", "reasoning": "fmt.Sprintf allocates a new string each iteration; with 10k+ items this causes significant GC pressure"}`,
 		},
 		{
 			Name: "correctness",
@@ -80,7 +90,10 @@ func AllConcerns() []Concern {
 - Are API contracts and interfaces respected?
 - Are invariants maintained across the change?
 - Could the change break existing callers?
-- Are defer/cleanup operations in the correct order?`,
+- Are defer/cleanup operations in the correct order?
+
+Example of a correct finding:
+{"file": "repo.go", "line": 34, "severity": "high", "message": "Unchecked error: os.Remove return value discarded, file may not be deleted", "fix": "if err := os.Remove(path); err != nil { return fmt.Errorf(\"cleanup failed: %w\", err) }", "reasoning": "os.Remove can fail due to permissions or file-in-use; ignoring the error leaves stale temp files that accumulate over time"}`,
 		},
 		{
 			Name: "style",
@@ -91,7 +104,10 @@ func AllConcerns() []Concern {
 - Abstraction level: is the code at a consistent abstraction level?
 - Error messages: are they actionable and specific?
 - API design: is the public surface minimal and intuitive?
-- Testability: is the code structured for easy testing?`,
+- Testability: is the code structured for easy testing?
+
+Example of a correct finding:
+{"file": "client.go", "line": 12, "severity": "low", "message": "Exported function FetchData lacks doc comment", "fix": "// FetchData retrieves data from the remote API by ID.\nfunc FetchData(id string) (*Data, error) {", "reasoning": "Go convention requires doc comments on all exported symbols; godoc and linters flag this"}`,
 		},
 	}
 }

internal/review/prompt.go

@@ -2,6 +2,8 @@ package review
 
 import (
 	"fmt"
+	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/GrayCodeAI/sight/internal/diff"
@@ -22,7 +24,8 @@ IMPORTANT: Respond ONLY with a JSON array of findings. Each finding must have:
   "severity": "critical|high|medium|low|info",
   "message": "Clear description of the issue",
   "fix": "Suggested code fix or approach",
-  "reasoning": "Why this is a problem"
+  "reasoning": "Why this is a problem",
+  "cwe": "CWE-89 (if applicable, otherwise empty string)"
 }
 
 If you find no issues, respond with an empty array: []
@@ -82,6 +85,89 @@ func BuildPrompt(concern Concern, files []diff.File, contextLines int) string {
 	return b.String()
 }
 
+// detectLanguages counts file extensions in the diff and returns a formatted
+// language context summary. The primary language is the one with the most files.
+func detectLanguages(files []diff.File) string {
+	extCount := make(map[string]int)
+	for _, f := range files {
+		if f.Deleted {
+			continue
+		}
+		ext := filepath.Ext(f.Path)
+		if ext == "" {
+			ext = filepath.Base(f.Path)
+		}
+		extCount[ext]++
+	}
+	if len(extCount) == 0 {
+		return ""
+	}
+
+	// Determine primary language by highest count.
+	type extEntry struct {
+		ext   string
+		count int
+	}
+	entries := make([]extEntry, 0, len(extCount))
+	for ext, count := range extCount {
+		entries = append(entries, extEntry{ext, count})
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		if entries[i].count != entries[j].count {
+			return entries[i].count > entries[j].count
+		}
+		return entries[i].ext < entries[j].ext
+	})
+
+	// Map extensions to language names for primary language display.
+	langNames := map[string]string{
+		".go":    "Go",
+		".py":    "Python",
+		".js":    "JavaScript",
+		".ts":    "TypeScript",
+		".tsx":   "TypeScript (React)",
+		".jsx":   "JavaScript (React)",
+		".java":  "Java",
+		".rs":    "Rust",
+		".rb":    "Ruby",
+		".cpp":   "C++",
+		".c":     "C",
+		".h":     "C/C++ Header",
+		".cs":    "C#",
+		".php":   "PHP",
+		".swift": "Swift",
+		".kt":    "Kotlin",
+		".scala": "Scala",
+		".yaml":  "YAML",
+		".yml":   "YAML",
+		".json":  "JSON",
+		".toml":  "TOML",
+		".md":    "Markdown",
+		".sql":   "SQL",
+		".sh":    "Shell",
+		".bash":  "Bash",
+	}
+
+	primary := entries[0].ext
+	primaryLang := langNames[primary]
+	if primaryLang == "" {
+		primaryLang = strings.TrimPrefix(primary, ".")
+	}
+
+	var b strings.Builder
+	b.WriteString("## Context\n")
+	b.WriteString(fmt.Sprintf("Primary language: %s\n", primaryLang))
+	b.WriteString("Files: ")
+	for i, e := range entries {
+		if i > 0 {
+			b.WriteString(", ")
+		}
+		b.WriteString(fmt.Sprintf("%d %s", e.count, e.ext))
+	}
+	b.WriteString("\n")
+	return b.String()
+}
+
 // BuildPromptEnhanced constructs a PR-Agent style prompt that separates new and
 // old hunks with clear section markers. This helps the LLM distinguish added
 // code from removed code more accurately.
@@ -90,6 +176,12 @@ func BuildPromptEnhanced(concern Concern, files []diff.File, contextLines int) s
 
 	b.WriteString("Review the following code changes:\n\n")
 
+	// Inject language context before the diff content.
+	if langCtx := detectLanguages(files); langCtx != "" {
+		b.WriteString(langCtx)
+		b.WriteString("\n")
+	}
+
 	for _, file := range files {
 		if file.Deleted {
 			continue