Fix:read file encoding

Author: aheizi

src/core/tools/applyDiffTool.ts

@@ -1,5 +1,5 @@
 import path from "path"
-import fs from "fs/promises"
+import { readFileSmart } from "../../integrations/misc/readFileWithEncoding"
 
 import { TelemetryService } from "@roo-code/telemetry"
 
@@ -87,7 +87,7 @@ export async function applyDiffTool(
 				return
 			}
 
-			const originalContent = await fs.readFile(absolutePath, "utf-8")
+			const originalContent = await readFileSmart(absolutePath)
 
 			// Apply the diff to the original content
 			const diffResult = (await cline.diffStrategy?.applyDiff(

src/core/tools/insertContentTool.ts

@@ -1,5 +1,5 @@
 import delay from "delay"
-import fs from "fs/promises"
+import { readFileSmart } from "../../integrations/misc/readFileWithEncoding"
 import path from "path"
 
 import { getReadablePath } from "../../utils/path"
@@ -89,7 +89,7 @@ export async function insertContentTool(
 		cline.consecutiveMistakeCount = 0
 
 		// Read the file
-		const fileContent = await fs.readFile(absolutePath, "utf8")
+		const fileContent = await readFileSmart(absolutePath)
 		cline.diffViewProvider.editType = "modify"
 		cline.diffViewProvider.originalContent = fileContent
 		const lines = fileContent.split("\n")

src/core/tools/searchAndReplaceTool.ts

@@ -1,6 +1,6 @@
 // Core Node.js imports
 import path from "path"
-import fs from "fs/promises"
+import { readFileSmart } from "../../integrations/misc/readFileWithEncoding"
 import delay from "delay"
 
 // Internal imports
@@ -143,7 +143,7 @@ export async function searchAndReplaceTool(
 		// Read and process file content
 		let fileContent: string
 		try {
-			fileContent = await fs.readFile(absolutePath, "utf-8")
+			fileContent = await readFileSmart(absolutePath)
 		} catch (error) {
 			cline.consecutiveMistakeCount++
 			cline.recordToolError("search_and_replace")

src/integrations/editor/DiffViewProvider.ts

@@ -8,6 +8,7 @@ import { createDirectoriesForFile } from "../../utils/fs"
 import { arePathsEqual } from "../../utils/path"
 import { formatResponse } from "../../core/prompts/responses"
 import { diagnosticsToProblemsString, getNewDiagnostics } from "../diagnostics"
+import { readFileSmart } from "../misc/readFileWithEncoding"
 
 import { DecorationController } from "./DecorationController"
 
@@ -53,7 +54,7 @@ export class DiffViewProvider {
 		this.preDiagnostics = vscode.languages.getDiagnostics()
 
 		if (fileExists) {
-			this.originalContent = await fs.readFile(absolutePath, "utf-8")
+			this.originalContent = await readFileSmart(absolutePath)
 		} else {
 			this.originalContent = ""
 		}

src/integrations/misc/__tests__/readFileWithEncoding.test.ts

@@ -0,0 +1,120 @@
+import { readFileSmart, scoreText, getCandidateEncodings, tryDecodeBuffer } from "../readFileWithEncoding"
+import * as fs from "fs/promises"
+import * as chardet from "chardet"
+import * as iconv from "iconv-lite"
+
+jest.mock("fs/promises")
+jest.mock("chardet")
+jest.mock("iconv-lite")
+
+const mockedFs = fs as jest.Mocked<typeof fs>
+const mockedChardet = chardet as jest.Mocked<typeof chardet>
+const mockedIconv = iconv as jest.Mocked<typeof iconv>
+
+describe("readFileWithEncoding", () => {
+	describe("scoreText", () => {
+		it("should score pure ASCII text lower", () => {
+			const score = scoreText("hello world")
+			expect(score).toBeLessThan(0)
+		})
+
+		it("should score Chinese text higher", () => {
+			const score = scoreText("你好世界")
+			expect(score).toBeGreaterThan(0)
+		})
+
+		it("should score mixed text appropriately", () => {
+			const score = scoreText("hello 你好")
+			expect(score).toBeGreaterThan(0)
+		})
+	})
+
+	describe("getCandidateEncodings", () => {
+		it("should include base encodings", () => {
+			const encodings = getCandidateEncodings("")
+			expect(encodings).toEqual(expect.arrayContaining(["utf-8", "gb18030", "gbk", "shift_jis"]))
+		})
+
+		it("should add detected encoding", () => {
+			const encodings = getCandidateEncodings("big5")
+			expect(encodings).toContain("big5")
+		})
+	})
+
+	describe("tryDecodeBuffer", () => {
+		it("should return best decoded text", () => {
+			const mockResult = {
+				text: "测试内容",
+				score: 1.5,
+				encoding: "gbk",
+			}
+			jest.spyOn(require("../readFileWithEncoding"), "tryDecodeBuffer").mockReturnValue(mockResult)
+
+			const buffer = Buffer.from("测试内容")
+			const result = tryDecodeBuffer(buffer, ["utf-8", "gbk", "shift_jis"])
+
+			expect(result.encoding).toBe("gbk")
+			expect(result.text).toBe("测试内容")
+			expect(result.score).toBe(1.5)
+		})
+	})
+
+	describe("readFileSmart", () => {
+		beforeEach(() => {
+			jest.clearAllMocks()
+		})
+
+		it("should read utf-8 text file correctly", async () => {
+			const mockBuffer = Buffer.from("utf8内容")
+			mockedFs.readFile.mockResolvedValue(mockBuffer)
+			mockedChardet.detect.mockReturnValue("utf-8")
+			mockedIconv.decode.mockImplementation((buffer: Buffer) => buffer.toString())
+
+			const result = await readFileSmart("test.txt")
+			expect(result).toBe("utf8内容")
+		})
+
+		it("should read gbk text file correctly", async () => {
+			const mockBuffer = Buffer.from("gbk内容")
+			mockedFs.readFile.mockResolvedValue(mockBuffer)
+			mockedChardet.detect.mockReturnValue("gbk")
+			mockedIconv.decode.mockImplementation((buffer: Buffer, encoding: string) =>
+				encoding === "gbk" ? "gbk解码内容" : buffer.toString(),
+			)
+
+			const result = await readFileSmart("test.txt")
+			expect(result).toBe("gbk解码内容")
+		})
+
+		it("should force utf-8 output when toUtf8=true", async () => {
+			const mockBuffer = Buffer.from("gbk内容")
+			mockedFs.readFile.mockResolvedValue(mockBuffer)
+			mockedChardet.detect.mockReturnValue("gbk")
+			mockedIconv.decode.mockReturnValue("gbk解码内容")
+
+			const result = await readFileSmart("test.txt", true)
+			expect(result).toBe(Buffer.from("gbk解码内容").toString("utf8"))
+		})
+
+		it("should fallback to utf-8 when decode fails", async () => {
+			const mockBuffer = Buffer.from("fallback内容")
+			mockedFs.readFile.mockResolvedValue(mockBuffer)
+			mockedChardet.detect.mockReturnValue("unknown")
+			mockedIconv.decode.mockImplementation(() => {
+				throw new Error("Decode error")
+			})
+
+			const result = await readFileSmart("test.bin")
+			expect(result).toBe("fallback内容")
+		})
+
+		it("should handle text file extensions specially", async () => {
+			const mockBuffer = Buffer.from("md内容")
+			mockedFs.readFile.mockResolvedValue(mockBuffer)
+			mockedChardet.detect.mockReturnValue("utf-8")
+
+			const result = await readFileSmart("test.md")
+			expect(result).toBe("md内容")
+		})
+	})
+})

src/integrations/misc/common-constants.ts

@@ -0,0 +1,55 @@
+/**
+ * Common text file extensions that should always be treated as text
+ * Stored without leading dot for flexibility
+ */
+export const TEXT_FILE_EXTENSIONS = [
+	"txt",
+	"md",
+	"log",
+	"rst",
+	"tex",
+	"json",
+	"yaml",
+	"yml",
+	"ini",
+	"conf",
+	"cfg",
+	"env",
+	"toml",
+	"properties",
+	"js",
+	"jsx",
+	"ts",
+	"tsx",
+	"c",
+	"cpp",
+	"h",
+	"hpp",
+	"java",
+	"py",
+	"rb",
+	"php",
+	"go",
+	"rs",
+	"swift",
+	"scala",
+	"pl",
+	"lua",
+	"sh",
+	"bat",
+	"ps1",
+	"html",
+	"xml",
+	"css",
+	"scss",
+	"less",
+	"styl",
+	"vue",
+	"csv",
+	"tsv",
+	"cs",
+	"kt",
+	"m",
+	"r",
+	"dart",
+]

src/integrations/misc/extract-text.ts

@@ -4,6 +4,8 @@ import pdf from "pdf-parse/lib/pdf-parse"
 import mammoth from "mammoth"
 import fs from "fs/promises"
 import { isBinaryFile } from "isbinaryfile"
+import { readFileSmart } from "./readFileWithEncoding"
+import { TEXT_FILE_EXTENSIONS } from "./common-constants"
 
 export async function extractTextFromFile(filePath: string): Promise<string> {
 	try {
@@ -13,23 +15,25 @@ export async function extractTextFromFile(filePath: string): Promise<string> {
 	}
 
 	const fileExtension = path.extname(filePath).toLowerCase()
-
+	// Convert extensions from common constants to include leading dot
+	const textExtensionsWithDot = TEXT_FILE_EXTENSIONS.map((ext) => `.${ext}`)
 	switch (fileExtension) {
 		case ".pdf":
 			return extractTextFromPDF(filePath)
 		case ".docx":
 			return extractTextFromDOCX(filePath)
 		case ".ipynb":
 			return extractTextFromIPYNB(filePath)
-		default: {
+		default:
+			if (textExtensionsWithDot.includes(fileExtension)) {
+				return addLineNumbers(await readFileSmart(filePath, true))
+			}
 			const isBinary = await isBinaryFile(filePath).catch(() => false)
-
 			if (!isBinary) {
-				return addLineNumbers(await fs.readFile(filePath, "utf8"))
+				return addLineNumbers(await readFileSmart(filePath, true))
 			} else {
 				throw new Error(`Cannot read text for file type: ${fileExtension}`)
 			}
-		}
 	}
 }