Fix codebase indexing for plain text files (#938)

* fix(code-index): index plain text files

* fix(tree-sitter): skip plain text definition parsing

* fix(code-index): address plain text review feedback

* fix(tree-sitter): preserve Scala structural parsing

* test(code-index): remove redundant async usage

Author: WebMad

src/services/code-index/processors/__tests__/parser.spec.ts

@@ -267,6 +267,31 @@ describe("CodeParser", () => {
 	})
 
 	describe("_chunkTextByLines", () => {
+		it("should not emit a chunk whose segment hash has already been seen", () => {
+			const lines = ["Fallback content long enough to produce a chunk without being filtered out."]
+			const seenSegmentHashes = new Set<string>()
+
+			const firstResult = parser["_chunkTextByLines"](
+				lines,
+				"manual.txt",
+				"hash",
+				"fallback_chunk",
+				seenSegmentHashes,
+			)
+			const duplicateResult = parser["_chunkTextByLines"](
+				lines,
+				"manual.txt",
+				"hash",
+				"fallback_chunk",
+				seenSegmentHashes,
+			)
+
+			expect(firstResult).toHaveLength(1)
+			expect(firstResult[0].segmentHash).toMatch(/^[a-f0-9]{64}$/)
+			expect(seenSegmentHashes).toEqual(new Set([firstResult[0].segmentHash]))
+			expect(duplicateResult).toEqual([])
+		})
+
 		it("should handle oversized lines by splitting them", async () => {
 			const longLine = "a".repeat(2000)
 			const lines = ["normal", longLine, "normal"]

src/services/code-index/processors/__tests__/parser.txt.spec.ts

@@ -0,0 +1,58 @@
+import { CodeParser } from "../parser"
+import { scannerExtensions, shouldUseFallbackChunking } from "../../shared/supported-extensions"
+
+vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({
+	TelemetryService: {
+		instance: {
+			captureEvent: vi.fn(),
+		},
+	},
+}))
+
+describe("CodeParser - plain text support", () => {
+	it("supports .txt files through fallback chunking", async () => {
+		expect(scannerExtensions).toContain(".txt")
+		expect(shouldUseFallbackChunking(".txt")).toBe(true)
+
+		const content = [
+			"Zoo Code plain text indexing regression test.",
+			"This sentence contains searchable content that only exists in the text file.",
+			"The fallback parser should preserve every line while creating an indexable chunk.",
+		].join("\n")
+
+		const blocks = await new CodeParser().parseFile("manual.txt", {
+			content,
+			fileHash: "txt-file-hash",
+		})
+
+		expect(blocks).toHaveLength(1)
+		expect(blocks[0]).toMatchObject({
+			file_path: "manual.txt",
+			type: "fallback_chunk",
+			start_line: 1,
+			end_line: 3,
+			content,
+			fileHash: "txt-file-hash",
+			segmentHash: expect.any(String),
+		})
+	})
+
+	it("parses uppercase .TXT extensions", async () => {
+		expect(shouldUseFallbackChunking(".TXT")).toBe(true)
+
+		const content = "Uppercase plain-text extension content long enough to produce a fallback chunk."
+		const blocks = await new CodeParser().parseFile("manual.TXT", {
+			content,
+			fileHash: "uppercase-txt-file-hash",
+		})
+
+		expect(blocks).toHaveLength(1)
+		expect(blocks[0]).toMatchObject({
+			file_path: "manual.TXT",
+			type: "fallback_chunk",
+			content,
+			fileHash: "uppercase-txt-file-hash",
+			segmentHash: expect.any(String),
+		})
+	})
+})

src/services/code-index/processors/parser.ts

@@ -45,7 +45,7 @@ export class CodeParser implements ICodeParser {
 		let content: string
 		let fileHash: string
 
-		if (options?.content) {
+		if (options?.content !== undefined) {
 			content = options.content
 			fileHash = options.fileHash || this.createFileHash(content)
 		} else {

src/services/code-index/shared/supported-extensions.ts

@@ -1,4 +1,7 @@
 import { extensions as allExtensions } from "../../tree-sitter"
+import { fallbackExtensions, isFallbackExtension } from "../../shared/fallback-extensions"
+
+export { fallbackExtensions }
 
 // Include all extensions including markdown for the scanner
 export const scannerExtensions = allExtensions
@@ -18,17 +21,11 @@ export const scannerExtensions = allExtensions
  *
  * Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere
  */
-export const fallbackExtensions = [
-	".vb", // Visual Basic .NET - no dedicated WASM parser
-	".scala", // Scala - uses fallback chunking instead of Lua query workaround
-	".swift", // Swift - uses fallback chunking due to parser instability
-]
-
 /**
  * Check if a file extension should use fallback chunking
  * @param extension File extension (including the dot)
  * @returns true if the extension should use fallback chunking
  */
 export function shouldUseFallbackChunking(extension: string): boolean {
-	return fallbackExtensions.includes(extension.toLowerCase())
+	return isFallbackExtension(extension)
 }

src/services/shared/fallback-extensions.ts

@@ -0,0 +1,29 @@
+/**
+ * Extensions that should not be parsed for structural definitions and should
+ * instead use line-based fallback chunking where indexing is supported.
+ */
+export const fallbackExtensions = [".txt", ".vb", ".scala", ".swift"] as const
+
+/**
+ * Fallback extensions that do not have a structural parser. Scala and Swift
+ * still support structural parsing outside code indexing.
+ */
+export const nonStructuralExtensions = [".txt", ".vb"] as const
+
+/**
+ * Check whether a file extension should bypass structural parsing.
+ *
+ * @param extension File extension, including the leading dot
+ */
+export function isFallbackExtension(extension: string): boolean {
+	return (fallbackExtensions as readonly string[]).includes(extension.toLowerCase())
+}
+
+/**
+ * Check whether a file extension should bypass structural parsing entirely.
+ *
+ * @param extension File extension, including the leading dot
+ */
+export function isNonStructuralExtension(extension: string): boolean {
+	return (nonStructuralExtensions as readonly string[]).includes(extension.toLowerCase())
+}

src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts

@@ -0,0 +1,37 @@
+// Mocks must come first, before imports
+
+vi.mock("fs/promises", () => ({
+	readFile: vi.fn(),
+}))
+
+vi.mock("../../../utils/fs", () => ({
+	fileExistsAtPath: vi.fn().mockResolvedValue(true),
+}))
+
+vi.mock("../languageParser", () => ({
+	loadRequiredLanguageParsers: vi.fn(),
+}))
+
+import * as fs from "fs/promises"
+import { loadRequiredLanguageParsers } from "../languageParser"
+import { parseSourceCodeDefinitionsForFile } from "../index"
+
+describe("Non-structural Extension Integration Tests", () => {
+	beforeEach(() => {
+		vi.clearAllMocks()
+	})
+
+	it.each(["manual.txt", "legacy.vb"])(
+		"returns undefined for %s without loading a tree-sitter parser",
+		async (filePath) => {
+			const result = await parseSourceCodeDefinitionsForFile(filePath)
+
+			expect(result).toBeUndefined()
+		},
+	)
+
+	afterEach(() => {
+		expect(loadRequiredLanguageParsers).not.toHaveBeenCalled()
+		expect(fs.readFile).not.toHaveBeenCalled()
+	})
+})

src/services/tree-sitter/index.ts

@@ -5,6 +5,7 @@ import { fileExistsAtPath } from "../../utils/fs"
 import { parseMarkdown } from "./markdownParser"
 import { RooIgnoreController } from "../../core/ignore/RooIgnoreController"
 import { QueryCapture } from "web-tree-sitter"
+import { isNonStructuralExtension } from "../shared/fallback-extensions"
 
 // Private constant
 const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4
@@ -66,6 +67,8 @@ const extensions = [
 	// Markdown
 	"md",
 	"markdown",
+	// Plain text
+	"txt",
 	// JSON
 	"json",
 	// CSS
@@ -111,6 +114,11 @@ export async function parseSourceCodeDefinitionsForFile(
 		return undefined
 	}
 
+	// Files without a structural parser have no definitions to extract
+	if (isNonStructuralExtension(ext)) {
+		return undefined
+	}
+
 	// Special case for markdown files
 	if (ext === ".md" || ext === ".markdown") {
 		// Check if we have permission to access this file