fix: encoding detection

Author: bajrangCoder

src/utils/encodings.js

@@ -55,6 +55,40 @@ function detectBOM(bytes) {
 	return null;
 }
 
+function isValidUTF8(bytes) {
+	let i = 0;
+	while (i < bytes.length) {
+		const byte = bytes[i];
+
+		if (byte < 0x80) {
+			i++;
+		} else if (byte >> 5 === 0x06) {
+			if (i + 1 >= bytes.length || bytes[i + 1] >> 6 !== 0x02) return false;
+			i += 2;
+		} else if (byte >> 4 === 0x0e) {
+			if (
+				i + 2 >= bytes.length ||
+				bytes[i + 1] >> 6 !== 0x02 ||
+				bytes[i + 2] >> 6 !== 0x02
+			)
+				return false;
+			i += 3;
+		} else if (byte >> 3 === 0x1e) {
+			if (
+				i + 3 >= bytes.length ||
+				bytes[i + 1] >> 6 !== 0x02 ||
+				bytes[i + 2] >> 6 !== 0x02 ||
+				bytes[i + 3] >> 6 !== 0x02
+			)
+				return false;
+			i += 4;
+		} else {
+			return false;
+		}
+	}
+	return true;
+}
+
 export async function detectEncoding(buffer) {
 	if (!buffer || buffer.byteLength === 0) {
 		return settings.value.defaultFileEncoding || "UTF-8";
@@ -74,9 +108,10 @@ export async function detectEncoding(buffer) {
 		else if (byte < 0x80) ascii++;
 	}
 
-	if (ascii / sample.length > 0.95) return "UTF-8";
 	if (nulls > sample.length * 0.3) return "UTF-16LE";
 
+	if (isValidUTF8(sample)) return "UTF-8";
+
 	const encodings = [
 		...new Set([
 			"UTF-8",