From 52e4de20163528b7cde53b17ea6c5175f98cf22c Mon Sep 17 00:00:00 2001
From: Shun-ichi Goto <gotoh@taiyo.co.jp>
Date: Thu, 18 Mar 2021 12:56:00 +0900
Subject: [PATCH] Support auto detection of text encoding for CSV and TSV

Read CSV/TSV file with detected encoding by using Ude (Mozilla Universal Charset Detector).
If data is not enough to detect, use system default encoding is used.
New package "Ude" is added to ExcelMerge project.
---
 ExcelMerge/CsvReader.cs      | 9 ++++++++-
 ExcelMerge/ExcelMerge.csproj | 3 +++
 ExcelMerge/TsvReader.cs      | 8 +++++++-
 ExcelMerge/packages.config   | 1 +
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/ExcelMerge/CsvReader.cs b/ExcelMerge/CsvReader.cs
index 5f84dfe..84a77ba 100644
--- a/ExcelMerge/CsvReader.cs
+++ b/ExcelMerge/CsvReader.cs
@@ -8,8 +8,14 @@ internal class CsvReader
     {
         internal static IEnumerable<ExcelRow> Read(string path)
         {
-            using (var sr = new StreamReader(path, Encoding.UTF8))
+            using (var stream = File.OpenRead(path))
             {
+                var detector = new Ude.CharsetDetector();
+                detector.Feed(stream);
+                detector.DataEnd();
+                var encoding = detector.IsDone() ? Encoding.GetEncoding(detector.Charset) : Encoding.Default;
+                stream.Position = 0;
+                var sr = new StreamReader(stream, encoding);
                 var rowIndex = 0;
                 while (!sr.EndOfStream)
                 {
@@ -22,5 +28,6 @@ internal static IEnumerable<ExcelRow> Read(string path)
                 }
             }
         }
+
     }
 }
diff --git a/ExcelMerge/ExcelMerge.csproj b/ExcelMerge/ExcelMerge.csproj
index bb3f1f4..3eb5c2b 100644
--- a/ExcelMerge/ExcelMerge.csproj
+++ b/ExcelMerge/ExcelMerge.csproj
@@ -60,6 +60,9 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
+    <Reference Include="Ude.NetStandard, Version=1.0.2.0, Culture=neutral, PublicKeyToken=103cb45fc06c90e4, processorArchitecture=MSIL">
+      <HintPath>..\packages\Ude.NetStandard.1.2.0\lib\net45\Ude.NetStandard.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="CsvReader.cs" />
diff --git a/ExcelMerge/TsvReader.cs b/ExcelMerge/TsvReader.cs
index d490e33..d64ddfa 100644
--- a/ExcelMerge/TsvReader.cs
+++ b/ExcelMerge/TsvReader.cs
@@ -8,8 +8,14 @@ public class TsvReader
     {
         internal static IEnumerable<ExcelRow> Read(string path)
         {
-            using (var sr = new StreamReader(path, Encoding.UTF8))
+            using (var stream = File.OpenRead(path))
             {
+                var detector = new Ude.CharsetDetector();
+                detector.Feed(stream);
+                detector.DataEnd();
+                var encoding = detector.IsDone() ? Encoding.GetEncoding(detector.Charset) : Encoding.Default;
+                stream.Position = 0;
+                var sr = new StreamReader(stream, encoding);
                 var rowIndex = 0;
                 while (!sr.EndOfStream)
                 {
diff --git a/ExcelMerge/packages.config b/ExcelMerge/packages.config
index 74b655d..18044b2 100644
--- a/ExcelMerge/packages.config
+++ b/ExcelMerge/packages.config
@@ -3,4 +3,5 @@
   <package id="Diff4Net" version="1.1.2" targetFramework="net452" />
   <package id="NPOI" version="2.3.0" targetFramework="net452" />
   <package id="SharpZipLib" version="0.86.0" targetFramework="net452" />
+  <package id="Ude.NetStandard" version="1.2.0" targetFramework="net452" />
 </packages>
\ No newline at end of file