Fix issue where line starts with a text qualifier (#54)

In Issue 53, wvdvegt discovered a bug where a line could start with a text qualifier and the state machine would incorrectly begin a new field rather than resume the previous field.

Author: tspence

CSVFile.nuspec

@@ -2,7 +2,7 @@
 <package >
 	<metadata>
 		<id>CSVFile</id>
-		<version>3.1.0</version>
+		<version>3.1.1</version>
 		<title>CSVFile</title>
 		<authors>Ted Spence</authors>
 		<owners>Ted Spence</owners>
@@ -13,16 +13,12 @@
 		<description>Tiny and fast CSV and TSV parsing library (40KB) with zero dependencies.  Compatible with most dot net versions.</description>
 		<icon>docs/icons8-spreadsheet-96.png</icon>
 		<releaseNotes>
-			April 4, 2022
+			March 7, 2023
 
-			* Improve handling for read-only properties with more errors and options
-			* Added support for excluding columns from serialization
-			* Added support for forcing columns to have text qualifiers
-			* Support for fully async serialization
-			* Unified all serialization code
+			* Fix issue when reading a stream with a text qualified field that ends with a newline
 		</releaseNotes>
 		<readme>docs/README.md</readme>
-		<copyright>Copyright 2006 - 2022</copyright>
+		<copyright>Copyright 2006 - 2023</copyright>
     	<tags>fast csv parser serialization deserialization streaming async</tags>
 		<repository type="git" url="https://github.com/tspence/csharp-csv-reader" />
 		<dependencies>

src/CSVStateMachine.cs

@@ -157,33 +157,22 @@ public string[] ParseChunk(string chunk, bool reachedEnd)
                 }
                 var c = _line[_position];
 
-                // Are we currently processing a text block (which may optionally span multiple lines)?
-                if (_inTextQualifier || (!_inTextQualifier && c == _settings.TextQualifier && _work.Length == 0))
+                // If we are resuming after starting a text qualifier, can we find the end?
+                if (_inTextQualifier)
                 {
-                    if (_inTextQualifier)
-                    {
-                        _work.Append(c);
-                    }
-                    _inTextQualifier = true;
-
-                    // Our next task is to find the end of this qualified-text field
                     var p2 = -1;
                     while (p2 < 0)
                     {
-
-                        // If we don't see an end in sight, read more from the stream
                         p2 = _line.IndexOf(_settings.TextQualifier, _position + 1);
                         if (p2 < 0)
                         {
-
-                            // No text qualifiers yet? Let's read more from the stream and continue
-                            _work.Append(_line.Substring(_position + 1));
-                            _line = string.Empty;
-                            _position = -1;
                             if (reachedEnd)
                             {
                                 State = CSVState.MissingTrailingQualifier;
                             }
+
+                            // Backtrack one character so we can move forward when the next chunk loads
+                            _position--;
                             return null;
                         }
 
@@ -203,6 +192,12 @@ public string[] ParseChunk(string chunk, bool reachedEnd)
                     // We're done parsing this text qualifier
                     _inTextQualifier = false;
                 }
+                // Is this the start of a text qualified field?
+                else if (c == _settings.TextQualifier && _work.Length == 0)
+                {
+                    _inTextQualifier = true;
+                    _position--;
+                }
                 // Are we at a line separator? Let's do a quick test first
                 else if (c == _settings.LineSeparator[0] && _position + _settings.LineSeparator.Length <= _line.Length)
                 {

tests/BasicParseTests.cs

@@ -190,5 +190,51 @@ public void ParseSepLineTest()
                 CSV.ParseSepLine("sep= this is a test since separators can't be more than a single character");
             });
         }
+        
+        [Test]
+        public void TestIssue53()
+        {
+            // This use case was reported by wvdvegt as https://github.com/tspence/csharp-csv-reader/issues/53
+            var line = CSV.ParseLine("\"test\",\"\n\",,,,\"Normal\",\"False\",,,\"Normal\",\"\"");
+            Assert.AreEqual("test", line[0]);
+            Assert.AreEqual("\n", line[1]);
+            Assert.AreEqual("", line[2]);
+            Assert.AreEqual("", line[3]);
+            Assert.AreEqual("", line[4]);
+            Assert.AreEqual("Normal", line[5]);
+            Assert.AreEqual("False", line[6]);
+            Assert.AreEqual("", line[7]);
+            Assert.AreEqual("", line[8]);
+            Assert.AreEqual("Normal", line[9]);
+            Assert.AreEqual("", line[10]);
+            
+            // Try same thing with MS-DOS newlines - CRLF
+            var line2 = CSV.ParseLine("\"test\",\"\r\n\",,,,\"Normal\",\"False\",,,\"Normal\",\"\"");
+            Assert.AreEqual("test", line2[0]);
+            Assert.AreEqual("\r\n", line2[1]);
+            Assert.AreEqual("", line2[2]);
+            Assert.AreEqual("", line2[3]);
+            Assert.AreEqual("", line2[4]);
+            Assert.AreEqual("Normal", line2[5]);
+            Assert.AreEqual("False", line2[6]);
+            Assert.AreEqual("", line2[7]);
+            Assert.AreEqual("", line2[8]);
+            Assert.AreEqual("Normal", line2[9]);
+            Assert.AreEqual("", line2[10]);
+            
+            // Try same thing with just LF only
+            var line3 = CSV.ParseLine("\"test\",\"\r\",,,,\"Normal\",\"False\",,,\"Normal\",\"\"");
+            Assert.AreEqual("test", line3[0]);
+            Assert.AreEqual("\r", line3[1]);
+            Assert.AreEqual("", line3[2]);
+            Assert.AreEqual("", line3[3]);
+            Assert.AreEqual("", line3[4]);
+            Assert.AreEqual("Normal", line3[5]);
+            Assert.AreEqual("False", line3[6]);
+            Assert.AreEqual("", line3[7]);
+            Assert.AreEqual("", line3[8]);
+            Assert.AreEqual("Normal", line3[9]);
+            Assert.AreEqual("", line3[10]);
+        }
     }
 }

tests/ReaderTest.cs

@@ -251,6 +251,36 @@ public void TestSepLineOverride()
             }
         }
 
+                
+        [Test]
+        public void TestIssue53()
+        {
+            var settings = new CSVSettings()
+            {
+                HeaderRowIncluded = false
+            };
+            
+            // This use case was reported by wvdvegt as https://github.com/tspence/csharp-csv-reader/issues/53
+            var source = "\"test\",\"" + Environment.NewLine + "\",,,,\"Normal\",\"False\",,,\"Normal\",\"\"";
+            using (var cr = CSVReader.FromString(source, settings))
+            {
+                foreach (var line in cr.Lines())
+                {
+                    Assert.AreEqual("test", line[0]);
+                    Assert.AreEqual(Environment.NewLine, line[1]);
+                    Assert.AreEqual("", line[2]);
+                    Assert.AreEqual("", line[3]);
+                    Assert.AreEqual("", line[4]);
+                    Assert.AreEqual("Normal", line[5]);
+                    Assert.AreEqual("False", line[6]);
+                    Assert.AreEqual("", line[7]);
+                    Assert.AreEqual("", line[8]);
+                    Assert.AreEqual("Normal", line[9]);
+                    Assert.AreEqual("", line[10]);
+                }
+            }
+        }
+
 #if HAS_ASYNC_IENUM
         [Test]
         public async Task TestAsyncReader()