feat: add support for VSCode file encodings

- Add iconv-lite dependency for encoding conversion
- Create src/utils/fileEncoding.ts utility that:
  - Gets file encoding from VSCode settings (files.encoding)
  - Provides readFileWithEncoding() and writeFileWithEncoding() functions
  - Includes fallback to UTF-8 if encoding conversion fails
- Update DiffViewProvider.ts to use encoding-aware file operations
- Update ApplyPatchTool.ts to use encoding-aware file operations
- Add comprehensive tests for the encoding utility

Fixes #11002

Author: roomote

pnpm-lock.yaml

@@ -5,6 +5,7 @@ import { type ClineSayTool, DEFAULT_WRITE_DELAY_MS } from "@roo-code/types"
 
 import { getReadablePath } from "../../utils/path"
 import { isPathOutsideWorkspace } from "../../utils/pathUtils"
+import { readFileWithEncoding, writeFileWithEncoding } from "../../utils/fileEncoding"
 import { Task } from "../task/Task"
 import { formatResponse } from "../prompts/responses"
 import { RecordSource } from "../context-tracking/FileContextTrackerTypes"
@@ -59,7 +60,8 @@ export class ApplyPatchTool extends BaseTool<"apply_patch"> {
 			// Process each hunk
 			const readFile = async (filePath: string): Promise<string> => {
 				const absolutePath = path.resolve(task.cwd, filePath)
-				return await fs.readFile(absolutePath, "utf8")
+				const { content } = await readFileWithEncoding(absolutePath)
+				return content
 			}
 
 			let changes: ApplyPatchFileChange[]
@@ -387,10 +389,10 @@ export class ApplyPatchTool extends BaseTool<"apply_patch"> {
 					writeDelayMs,
 				)
 			} else {
-				// Write to new path and delete old file
+				// Write to new path and delete old file with proper encoding
 				const parentDir = path.dirname(moveAbsolutePath)
 				await fs.mkdir(parentDir, { recursive: true })
-				await fs.writeFile(moveAbsolutePath, newContent, "utf8")
+				await writeFileWithEncoding(moveAbsolutePath, newContent)
 			}
 
 			// Delete the original file

src/core/tools/ApplyPatchTool.ts

@@ -8,6 +8,7 @@ import delay from "delay"
 import { type ClineSayTool, DEFAULT_WRITE_DELAY_MS } from "@roo-code/types"
 
 import { createDirectoriesForFile } from "../../utils/fs"
+import { readFileWithEncoding, writeFileWithEncoding } from "../../utils/fileEncoding"
 import { arePathsEqual, getReadablePath } from "../../utils/path"
 import { formatResponse } from "../../core/prompts/responses"
 import { diagnosticsToProblemsString, getNewDiagnostics } from "../diagnostics"
@@ -67,7 +68,8 @@ export class DiffViewProvider {
 		this.preDiagnostics = vscode.languages.getDiagnostics()
 
 		if (fileExists) {
-			this.originalContent = await fs.readFile(absolutePath, "utf-8")
+			const { content } = await readFileWithEncoding(absolutePath)
+			this.originalContent = content
 		} else {
 			this.originalContent = ""
 		}
@@ -651,9 +653,9 @@ export class DiffViewProvider {
 		// Get diagnostics before editing the file
 		this.preDiagnostics = vscode.languages.getDiagnostics()
 
-		// Write the content directly to the file
+		// Write the content directly to the file with proper encoding
 		await createDirectoriesForFile(absolutePath)
-		await fs.writeFile(absolutePath, content, "utf-8")
+		await writeFileWithEncoding(absolutePath, content)
 
 		// Open the document to ensure diagnostics are loaded
 		// When openFile is false (PREVENT_FOCUS_DISRUPTION enabled), we only open in memory

src/integrations/editor/DiffViewProvider.ts

@@ -479,6 +479,7 @@
 		"fastest-levenshtein": "^1.0.16",
 		"fzf": "^0.5.2",
 		"get-folder-size": "^5.0.0",
+		"iconv-lite": "^0.6.3",
 		"global-agent": "^3.0.0",
 		"google-auth-library": "^9.15.1",
 		"gray-matter": "^4.0.3",

src/package.json

@@ -0,0 +1,272 @@
+import * as vscode from "vscode"
+import * as fs from "fs/promises"
+import * as iconv from "iconv-lite"
+import {
+	getFileEncoding,
+	normalizeEncoding,
+	isEncodingSupported,
+	readFileWithEncoding,
+	writeFileWithEncoding,
+} from "../fileEncoding"
+
+// Mock vscode module
+vi.mock("vscode", () => ({
+	workspace: {
+		getConfiguration: vi.fn(),
+	},
+	Uri: {
+		file: vi.fn((path: string) => ({ fsPath: path })),
+	},
+}))
+
+// Mock fs/promises module
+vi.mock("fs/promises", () => ({
+	default: {
+		readFile: vi.fn(),
+		writeFile: vi.fn(),
+	},
+	readFile: vi.fn(),
+	writeFile: vi.fn(),
+}))
+
+// Mock iconv-lite module
+vi.mock("iconv-lite", () => ({
+	default: {
+		encodingExists: vi.fn(),
+		decode: vi.fn(),
+		encode: vi.fn(),
+	},
+	encodingExists: vi.fn(),
+	decode: vi.fn(),
+	encode: vi.fn(),
+}))
+
+describe("fileEncoding", () => {
+	const mockedVscode = vi.mocked(vscode)
+	const mockedFs = vi.mocked(fs)
+	const mockedIconv = vi.mocked(iconv)
+
+	beforeEach(() => {
+		vi.clearAllMocks()
+	})
+
+	describe("getFileEncoding", () => {
+		it("should return the configured encoding from VSCode settings", () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("cp852"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			const encoding = getFileEncoding("/path/to/file.txt")
+
+			expect(mockedVscode.Uri.file).toHaveBeenCalledWith("/path/to/file.txt")
+			expect(mockedVscode.workspace.getConfiguration).toHaveBeenCalledWith("files", {
+				fsPath: "/path/to/file.txt",
+			})
+			expect(mockConfig.get).toHaveBeenCalledWith("encoding", "utf8")
+			expect(encoding).toBe("cp852")
+		})
+
+		it("should return utf8 as default if no encoding is configured", () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("utf8"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			const encoding = getFileEncoding("/path/to/file.txt")
+
+			expect(encoding).toBe("utf8")
+		})
+	})
+
+	describe("normalizeEncoding", () => {
+		it("should normalize utf-8 to utf8", () => {
+			expect(normalizeEncoding("utf-8")).toBe("utf8")
+			expect(normalizeEncoding("UTF-8")).toBe("utf8")
+		})
+
+		it("should normalize windows code pages", () => {
+			expect(normalizeEncoding("windows1252")).toBe("windows1252")
+			expect(normalizeEncoding("windows-1252")).toBe("windows1252")
+		})
+
+		it("should normalize DOS code pages", () => {
+			expect(normalizeEncoding("cp852")).toBe("cp852")
+			expect(normalizeEncoding("CP852")).toBe("cp852")
+		})
+
+		it("should normalize ISO encodings", () => {
+			expect(normalizeEncoding("iso88591")).toBe("iso88591")
+			expect(normalizeEncoding("iso-8859-1")).toBe("iso88591")
+		})
+
+		it("should return the original encoding if not in the map", () => {
+			expect(normalizeEncoding("unknown-encoding")).toBe("unknown-encoding")
+		})
+	})
+
+	describe("isEncodingSupported", () => {
+		it("should return true for supported encodings", () => {
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(true)
+
+			expect(isEncodingSupported("utf8")).toBe(true)
+			expect(mockedIconv.encodingExists).toHaveBeenCalledWith("utf8")
+		})
+
+		it("should return false for unsupported encodings", () => {
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(false)
+
+			expect(isEncodingSupported("unknown")).toBe(false)
+			expect(mockedIconv.encodingExists).toHaveBeenCalledWith("unknown")
+		})
+	})
+
+	describe("readFileWithEncoding", () => {
+		beforeEach(() => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("utf8"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+		})
+
+		it("should read file with UTF-8 encoding directly", async () => {
+			const mockBuffer = Buffer.from("Hello World", "utf8")
+			mockedFs.readFile = vi.fn().mockResolvedValue(mockBuffer)
+
+			const result = await readFileWithEncoding("/path/to/file.txt")
+
+			expect(result.content).toBe("Hello World")
+			expect(result.encoding).toBe("utf8")
+			expect(result.usedFallback).toBe(false)
+		})
+
+		it("should read file with CP852 encoding", async () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("cp852"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			const mockBuffer = Buffer.from([0x8d, 0x8f, 0xa7]) // Some CP852 bytes
+			mockedFs.readFile = vi.fn().mockResolvedValue(mockBuffer)
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(true)
+			mockedIconv.decode = vi.fn().mockReturnValue("čćž")
+
+			const result = await readFileWithEncoding("/path/to/file.txt")
+
+			expect(mockedIconv.decode).toHaveBeenCalledWith(mockBuffer, "cp852")
+			expect(result.content).toBe("čćž")
+			expect(result.encoding).toBe("cp852")
+			expect(result.usedFallback).toBe(false)
+		})
+
+		it("should fall back to UTF-8 if encoding is not supported", async () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("unsupported-encoding"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			const mockBuffer = Buffer.from("Hello World", "utf8")
+			mockedFs.readFile = vi.fn().mockResolvedValue(mockBuffer)
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(false)
+
+			const result = await readFileWithEncoding("/path/to/file.txt")
+
+			expect(result.content).toBe("Hello World")
+			expect(result.encoding).toBe("unsupported-encoding")
+			expect(result.usedFallback).toBe(true)
+		})
+
+		it("should fall back to UTF-8 if decoding fails", async () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("cp852"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			const mockBuffer = Buffer.from("Hello World", "utf8")
+			mockedFs.readFile = vi.fn().mockResolvedValue(mockBuffer)
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(true)
+			mockedIconv.decode = vi.fn().mockImplementation(() => {
+				throw new Error("Decoding failed")
+			})
+
+			const result = await readFileWithEncoding("/path/to/file.txt")
+
+			expect(result.content).toBe("Hello World")
+			expect(result.encoding).toBe("cp852")
+			expect(result.usedFallback).toBe(true)
+		})
+	})
+
+	describe("writeFileWithEncoding", () => {
+		beforeEach(() => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("utf8"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+		})
+
+		it("should write file with UTF-8 encoding directly", async () => {
+			mockedFs.writeFile = vi.fn().mockResolvedValue(undefined)
+
+			const result = await writeFileWithEncoding("/path/to/file.txt", "Hello World")
+
+			expect(mockedFs.writeFile).toHaveBeenCalledWith("/path/to/file.txt", "Hello World", "utf8")
+			expect(result.encoding).toBe("utf8")
+			expect(result.usedFallback).toBe(false)
+		})
+
+		it("should write file with CP852 encoding", async () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("cp852"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			const mockBuffer = Buffer.from([0x8d, 0x8f, 0xa7])
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(true)
+			mockedIconv.encode = vi.fn().mockReturnValue(mockBuffer)
+			mockedFs.writeFile = vi.fn().mockResolvedValue(undefined)
+
+			const result = await writeFileWithEncoding("/path/to/file.txt", "čćž")
+
+			expect(mockedIconv.encode).toHaveBeenCalledWith("čćž", "cp852")
+			expect(mockedFs.writeFile).toHaveBeenCalledWith("/path/to/file.txt", mockBuffer)
+			expect(result.encoding).toBe("cp852")
+			expect(result.usedFallback).toBe(false)
+		})
+
+		it("should fall back to UTF-8 if encoding is not supported", async () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("unsupported-encoding"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(false)
+			mockedFs.writeFile = vi.fn().mockResolvedValue(undefined)
+
+			const result = await writeFileWithEncoding("/path/to/file.txt", "Hello World")
+
+			expect(mockedFs.writeFile).toHaveBeenCalledWith("/path/to/file.txt", "Hello World", "utf8")
+			expect(result.encoding).toBe("unsupported-encoding")
+			expect(result.usedFallback).toBe(true)
+		})
+
+		it("should fall back to UTF-8 if encoding fails", async () => {
+			const mockConfig = {
+				get: vi.fn().mockReturnValue("cp852"),
+			}
+			mockedVscode.workspace.getConfiguration = vi.fn().mockReturnValue(mockConfig)
+
+			mockedIconv.encodingExists = vi.fn().mockReturnValue(true)
+			mockedIconv.encode = vi.fn().mockImplementation(() => {
+				throw new Error("Encoding failed")
+			})
+			mockedFs.writeFile = vi.fn().mockResolvedValue(undefined)
+
+			const result = await writeFileWithEncoding("/path/to/file.txt", "Hello World")
+
+			expect(mockedFs.writeFile).toHaveBeenCalledWith("/path/to/file.txt", "Hello World", "utf8")
+			expect(result.encoding).toBe("cp852")
+			expect(result.usedFallback).toBe(true)
+		})
+	})
+})