Fix issue when final character in field is escaped. (#221)

docs/Csv/Sylvan.Data.Csv.Releases.md

@@ -1,5 +1,8 @@
 # Sylvan.Data.Csv Release Notes
 
+_1.3.5_
+- Fixes a bug where fields could be incorrectly read when the final character was escaped when reading with `CsvStyle.Escaped`.
+
 _1.3.4_
 - Adds `CsvSchema.Dynamic` which treats CSV data as having "variant" type. This can be useful when processing
     CSV data where the field type might change from row to row.

source/Sylvan.Data.Csv.Tests/CsvDataReaderTests.cs

@@ -1856,6 +1856,57 @@ public void QuotedHeader()
 		Assert.Equal("3", csv.GetString(2));
 	}
 
+	[Fact]
+	public void FinalCharInCellIsEscaped()
+	{
+		using var reader = new StringReader(@"a\, \,,b,\,");
+		using var csvReader = CsvDataReader.Create(reader, new CsvDataReaderOptions
+		{
+			CsvStyle = CsvStyle.Escaped,
+			HasHeaders = false,
+			Escape = '\\',
+		});
+
+		csvReader.Read();
+		var value0 = csvReader.GetString(0);
+		var value1 = csvReader.GetString(1);
+		var value2 = csvReader.GetString(2);
+		Assert.Equal("a, ,", value0); // This will fail; will be "a, \" instead of "a, ,"
+		Assert.Equal("b", value1);
+		Assert.Equal(",", value2);
+	}
+
+	[Fact]
+	public void EscapeEOF()
+	{
+		using var reader = new StringReader("\\");
+		Assert.Throws<InvalidDataException>(() =>
+		{
+			using var csvReader = CsvDataReader.Create(reader, new CsvDataReaderOptions
+			{
+				CsvStyle = CsvStyle.Escaped,
+				HasHeaders = false,
+				Escape = '\\',
+			});
+		});
+	}
+
+	[Fact]
+	public void FinalCharInCellIsEscapeError()
+	{
+		using var reader = new StringReader("\\\\\\\n");
+		using var csvReader = CsvDataReader.Create(reader, new CsvDataReaderOptions
+		{
+			CsvStyle = CsvStyle.Escaped,
+			HasHeaders = false,
+			Escape = '\\',
+		});
+
+		csvReader.Read();
+		var value0 = csvReader.GetString(0);
+		Assert.Equal("\\\n", value0);
+	}
+
 
 #if NET6_0_OR_GREATER
 

source/Sylvan.Data.Csv/CsvDataReader.cs

@@ -679,6 +679,7 @@ ReadResult ReadField(int fieldIdx)
 						c = buffer[idx++]; // the escaped char
 						if (IsEndOfLine(c))
 						{
+							idx--;// "unconsume" the newline character, so that ConsumeLineEnd can process it.
 							// if the escape precede an EOL, we might have to consume 2 chars
 							var r = ConsumeLineEnd(buffer, ref idx);
 							if (r == ReadResult.Incomplete)
@@ -693,9 +694,8 @@ ReadResult ReadField(int fieldIdx)
 					{
 						if (atEndOfText)
 						{
-							// TODO: not sure what to do here.
-							escapeCount++;
-							break;
+							// there was nothing to escape
+							throw new InvalidDataException();
 						}
 						return ReadResult.Incomplete;
 					}
@@ -1591,7 +1591,6 @@ CharSpan GetFieldUnsafe(int ordinal)
 
 	CharSpan PrepareField(int offset, int len, int escapeCount)
 	{
-
 		bool inQuote = true; // we start inside the quotes
 
 		var eLen = len - escapeCount;
@@ -1606,29 +1605,36 @@ CharSpan PrepareField(int offset, int len, int escapeCount)
 
 		int i = 0;
 		int d = 0;
-		while (d < eLen)
+		while (i < len)
 		{
 			var c = buffer[offset + i++];
 			if (inQuote)
 			{
-				if (c == escape && i + 1 < len)
+				if (c == escape)
 				{
-					c = buffer[offset + i++];
-					if (c != quote && c != escape)
+					if (i < len)
 					{
-						if (quote == escape)
+						c = buffer[offset + i++];
+						if (c != quote && c != escape)
 						{
-							// the escape we just saw was actually the closing quote
-							// the remainder of the field will be added verbatim
-							inQuote = false;
+							if (quote == escape)
+							{
+								// the escape we just saw was actually the closing quote
+								// the remainder of the field will be added verbatim
+								inQuote = false;
+							}
 						}
 					}
+					else
+					{
+						throw new InvalidDataException();
+					}
 				}
 				else
 				if (c == quote)
 				{
 					// we've found the broken closing quote
-					// skip it.
+					// skip it.					
 					inQuote = false;
 					continue;
 				}

source/Sylvan.Data.Csv/Sylvan.Data.Csv.csproj

@@ -2,7 +2,7 @@
 
 	<PropertyGroup>
 		<TargetFrameworks>net6.0;netstandard2.1;netstandard2.0</TargetFrameworks>
-		<VersionPrefix>1.3.4</VersionPrefix>
+		<VersionPrefix>1.3.5</VersionPrefix>
 		<Description>A .NET library for reading and writing delimited CSV data.</Description>
 		<PackageTags>csv;delimited;data;datareader;datawriter;simd</PackageTags>
 		<Nullable>enable</Nullable>