Fix buffer read for windows style newlines (#67)

tspence · web-flow · commit 4355ed88fd99 · 2024-08-06T08:40:57.000-07:00
Implement Joel's test for complex large files.  Correct the counting of positions when a chunk ends with a `\r` and the next chunk begins with a `\n`.  Update to version 3.2.0.
diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml
@@ -27,19 +27,22 @@ jobs:
     - name: Build (Framework 2.0 Tests)
       run: msbuild ./tests/net20/tests.net20.csproj
     - name: Test (net20)
-      run: ./NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests/net20/bin/Debug/tests.net20.dll
+      working-directory: ./tests/net20/bin/Debug/
+      run: ../../../../NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests.net20.dll
     - name: Build (Framework 4.0)
       run: msbuild ./src/net40/src.net40.csproj
     - name: Build (Framework 4.0 Tests)
       run: msbuild ./tests/net40/tests.net40.csproj
     - name: Test (net40)
-      run: ./NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests/net40/bin/Debug/tests.net40.dll
+      working-directory: ./tests/net40/bin/Debug
+      run: ../../../../NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests.net40.dll
     - name: Build (Framework 4.5)
       run: msbuild ./src/net45/src.net45.csproj
     - name: Build (Framework 4.5 Tests)
       run: msbuild ./tests/net45/tests.net45.csproj
     - name: Test (net45)
-      run: ./NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests/net45/bin/Debug/tests.net45.dll
+      working-directory: ./tests/net45/bin/Debug/
+      run: ../../../../NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests.net45.dll
     - name: Build (DotNet Core 5.0 and NetStandard 2.0)
       run: dotnet build ./csharp-csv-reader.sln
     - name: Test (net50)
diff --git a/CSVFile.nuspec b/CSVFile.nuspec
@@ -2,7 +2,7 @@
 <package >
 	<metadata>
 		<id>CSVFile</id>
-		<version>3.1.4</version>
+		<version>3.2.0</version>
 		<title>CSVFile</title>
 		<authors>Ted Spence</authors>
 		<owners>Ted Spence</owners>
@@ -15,8 +15,8 @@
 		<releaseNotes>
 			August 5, 2024
 			
-			* Add serialization options for arrays and objects
-			* Fix bad deploy of 3.1.3
+			* Fix issue with Windows-style newlines crossing chunks found by @joelverhagen
+			* Fix issue with endless loops reported by @wvvegt
 		</releaseNotes>
 		<readme>docs/README.md</readme>
 		<copyright>Copyright 2006 - 2024</copyright>
diff --git a/src/CSV.cs b/src/CSV.cs
@@ -73,6 +73,10 @@ public static IEnumerable<string[]> ParseStream(StreamReader inStream, CSVSettin
                 {
                     yield return row;
                 } 
+                else if (inStream.EndOfStream)
+                {
+                    break;
+                }
             }
         }
 
@@ -101,6 +105,10 @@ public static async IAsyncEnumerable<string[]> ParseStreamAsync(StreamReader inS
                 {
                     yield return row;
                 }
+                else if (inStream.EndOfStream)
+                {
+                    break;
+                }
             }
         }
 #endif
diff --git a/src/CSVStateMachine.cs b/src/CSVStateMachine.cs
@@ -65,7 +65,7 @@ public class CSVStateMachine
         /// <returns></returns>
         public bool NeedsMoreText()
         {
-            return String.IsNullOrEmpty(_line) || _position >= _line.Length;
+            return String.IsNullOrEmpty(_line) || _position + _settings.LineSeparator.Length >= _line.Length;
         }
 
         /// <summary>
@@ -202,11 +202,13 @@ public string[] ParseChunk(string chunk, bool reachedEnd)
                     var notEnoughChars = _position + _settings.LineSeparator.Length > _line.Length;
                     if (notEnoughChars && !reachedEnd)
                     {
+                        // Backtrack one character so we can pick up the line separator completely next time
+                        _position--;
                         return null;
                     }
 
                     // If we have reached the end, but this isn't a complete line separator, it's just text
-                    if (notEnoughChars && reachedEnd)
+                    if (notEnoughChars)
                     {
                         _work.Append(c);
                     }
diff --git a/tests/PackageAssets.csv b/tests/PackageAssets.csv
diff --git a/tests/ReaderTest.cs b/tests/ReaderTest.cs
@@ -4,6 +4,10 @@
  * Home page: https://github.com/tspence/csharp-csv-reader
  */
 using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
 using NUnit.Framework;
 using CSVFile;
 #if HAS_ASYNC
@@ -336,6 +340,41 @@ public void TestMultipleNewlines()
             }
         }
 
+        [Test]
+        public void TestIssue62()
+        {
+            var inputLines = File.ReadAllLines("PackageAssets.csv");
+            var desiredLines = 53_543;
+            var linesToRead = Enumerable
+                .Repeat(inputLines, desiredLines / inputLines.Length + 1)
+                .SelectMany(x => x)
+                .Take(desiredLines)
+                .ToArray();
+
+            var config = new CSVSettings
+            {
+                HeaderRowIncluded = false,
+            };
+
+            var outputLines = 0;
+            var rawText = string.Join(Environment.NewLine, linesToRead);
+            var rawBytes = Encoding.UTF8.GetBytes(rawText);
+            using (var memoryStream = new MemoryStream(rawBytes))
+            {
+                using (var streamReader = new StreamReader(memoryStream))
+                {
+                    using (var csvReader = new CSVReader(streamReader, config))
+                    {
+                        foreach (var row in csvReader)
+                        {
+                            outputLines++;
+                        }
+                    }
+                }
+            }
+            Assert.AreEqual(desiredLines, outputLines);
+        }
+
 #if HAS_ASYNC_IENUM
         [Test]
         public async Task TestAsyncReader()
diff --git a/tests/net20/tests.net20.csproj b/tests/net20/tests.net20.csproj
@@ -74,6 +74,12 @@
       <Name>src.net20</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>
diff --git a/tests/net40/tests.net40.csproj b/tests/net40/tests.net40.csproj
@@ -71,6 +71,12 @@
       <Name>src.net40</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>
diff --git a/tests/net45/tests.net45.csproj b/tests/net45/tests.net45.csproj
@@ -71,6 +71,12 @@
       <Name>src.net45</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>
diff --git a/tests/net50/tests.net50.csproj b/tests/net50/tests.net50.csproj
@@ -30,4 +30,11 @@
     <ProjectReference Include="..\..\src\net50\src.net50.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
 </Project>
diff --git a/tests/net60/tests.net60.csproj b/tests/net60/tests.net60.csproj
@@ -32,5 +32,12 @@
     <ProjectReference Include="..\..\src\netstandard20\src.netstandard20.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
 </Project>
 

Original file line number	Diff line number	Diff line change
`@@ -73,6 +73,10 @@ public static IEnumerable<string[]> ParseStream(StreamReader inStream, CSVSettin`
`73`	`73`	`{`
`74`	`74`	`yield return row;`
`75`	`75`	`}`
	`76`	`+ else if (inStream.EndOfStream)`
	`77`	`+ {`
	`78`	`+ break;`
	`79`	`+ }`
`76`	`80`	`}`
`77`	`81`	`}`
`78`	`82`
`@@ -101,6 +105,10 @@ public static async IAsyncEnumerable<string[]> ParseStreamAsync(StreamReader inS`
`101`	`105`	`{`
`102`	`106`	`yield return row;`
`103`	`107`	`}`
	`108`	`+ else if (inStream.EndOfStream)`
	`109`	`+ {`
	`110`	`+ break;`
	`111`	`+ }`
`104`	`112`	`}`
`105`	`113`	`}`
`106`	`114`	`#endif`
Original file line number	Diff line number	Diff line change
`@@ -65,7 +65,7 @@ public class CSVStateMachine`
`65`	`65`	`/// <returns></returns>`
`66`	`66`	`public bool NeedsMoreText()`
`67`	`67`	`{`
`68`		`- return String.IsNullOrEmpty(_line) \|\| _position >= _line.Length;`
	`68`	`+ return String.IsNullOrEmpty(_line) \|\| _position + _settings.LineSeparator.Length >= _line.Length;`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`/// <summary>`
`@@ -202,11 +202,13 @@ public string[] ParseChunk(string chunk, bool reachedEnd)`
`202`	`202`	`var notEnoughChars = _position + _settings.LineSeparator.Length > _line.Length;`
`203`	`203`	`if (notEnoughChars && !reachedEnd)`
`204`	`204`	`{`
	`205`	`+ // Backtrack one character so we can pick up the line separator completely next time`
	`206`	`+ _position--;`
`205`	`207`	`return null;`
`206`	`208`	`}`
`207`	`209`
`208`	`210`	`// If we have reached the end, but this isn't a complete line separator, it's just text`
`209`		`- if (notEnoughChars && reachedEnd)`
	`211`	`+ if (notEnoughChars)`
`210`	`212`	`{`
`211`	`213`	`_work.Append(c);`
`212`	`214`	`}`