FlatFileDataReader not correctly ignoring ignored columns in several …

…places.

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.12.0 (2020-10-21)
+**Summary** - `FlatFileDataReader` not correctly ignoring ignored columns in several places.
+
+The ADO.NET classes didn't receive the same level of love that the rest of the library received when introducing ignored columns. When getting the column names, their ordinal positions, etc., the `FlatFileDataReader` was returning information for ignored columns. This caused the `DataTable` extensions to see too many column names and yet receive too few record values in the table (with the data shifted to the left for each missing column). Fixing the `FlatFileDataReader` fixed the `DataTable` problems, as well.
+
+Technically this is a breaking change that might warrant a major version change; however, as the previous behavior could not possibly be desired and few people actually use the ADO.NET classes, I am going to include this in the next minor version, treating it as just a bug fix.
+
 ## 4.11.0 (2020-10-09)
 **Summary* - Allow handling unrecognized rows when using schema selectors.
 

FlatFiles.Test/DataTableExtensionsTester.cs

@@ -1,8 +1,11 @@
 using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 using System.Data;
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -413,6 +416,39 @@ public void TestWriteFlatFile_ExtraColumn_Ignores()
 ", output);
         }
 
+        [TestMethod]
+        public void TestReadFlatFile_IgnoredColumns2()
+        {
+            const string data =
+@"A,B,C
+1,2,3
+4,5,6";
+            var schema = new SeparatedValueSchema();
+            schema.AddColumn(new StringColumn("A"));
+            schema.AddColumn(new IgnoredColumn("Ignored"));
+            schema.AddColumn(new StringColumn("C"));
+
+            var options = new SeparatedValueOptions()
+            {
+                IsFirstRecordSchema = true
+            };
+
+            var textReader = new StringReader(data);
+            var csvReader = new SeparatedValueReader(textReader, schema, options);
+
+            DataTable dataTable = new DataTable();
+            dataTable.ReadFlatFile(csvReader);
+            string[] columnNames = dataTable.Columns.OfType<DataColumn>()
+                .Select(r => r.ColumnName)
+                .ToArray();
+            CollectionAssert.AreEqual(new[] { "A", "C" }, columnNames);
+            Assert.AreEqual(2, dataTable.Rows.Count);
+            object[] values1 = dataTable.Rows[0].ItemArray;
+            CollectionAssert.AreEqual(new[] { "1", "3" }, values1);
+            object[] values2 = dataTable.Rows[1].ItemArray;
+            CollectionAssert.AreEqual(new[] { "4", "6" }, values2);
+        }
+
         [TestMethod]
         public void TestWriteFlatFile_IgnoredColumns()
         {

FlatFiles.Test/FlatFileDataReaderTester.cs

@@ -146,5 +146,48 @@ private static FlatFileDataReader GetFlatFileReader()
             var dataReader = new FlatFileDataReader(csvReader);
             return dataReader;
         }
+
+        [TestMethod]
+        public void TestFlatFileReader_IgnoreIgnoredColumns()
+        {
+            const string data =
+@"A,B,C
+1,2,3
+4,5,6";
+            var schema = new SeparatedValueSchema();
+            schema.AddColumn(new StringColumn("A"));
+            schema.AddColumn(new IgnoredColumn("Ignored"));
+            schema.AddColumn(new StringColumn("C"));
+
+            var options = new SeparatedValueOptions()
+            {
+                IsFirstRecordSchema = true
+            };
+
+            var textReader = new StringReader(data);
+            var csvReader = new SeparatedValueReader(textReader, schema, options);
+            using (var dataReader = new FlatFileDataReader(csvReader))
+            {
+                Assert.AreEqual("A", dataReader.GetName(0));
+                Assert.AreEqual("C", dataReader.GetName(1));
+                Assert.AreEqual(0, dataReader.GetOrdinal("A"));
+                Assert.AreEqual(-1, dataReader.GetOrdinal("B"));
+                Assert.AreEqual(1, dataReader.GetOrdinal("C"));
+
+                var schemaTable = dataReader.GetSchemaTable();
+                string[] columnNames = schemaTable.Rows.OfType<DataRow>()
+                    .Select(r => r.Field<string>("ColumnName"))
+                    .ToArray();
+                CollectionAssert.AreEqual(new[] { "A", "C" }, columnNames);
+
+                Assert.IsTrue(dataReader.Read());
+                object[] values1 = dataReader.GetValues();
+                CollectionAssert.AreEqual(new[] { "1", "3" }, values1);
+                Assert.IsTrue(dataReader.Read());
+                object[] values2 = dataReader.GetValues();
+                CollectionAssert.AreEqual(new[] { "4", "6" }, values2);
+                Assert.IsFalse(dataReader.Read());
+            }
+        }
     }
 }

FlatFiles/FlatFileDataReader.cs

@@ -3,6 +3,7 @@
 using System.Data;
 using System.Data.Common;
 using System.Globalization;
+using System.Linq;
 
 namespace FlatFiles
 {
@@ -60,6 +61,7 @@ public interface IFlatFileDataRecord : IDataRecord
     public sealed class FlatFileDataReader : IDataReader, IFlatFileDataRecord
     {
         private ISchema schema;  // cached
+        private ColumnCollection columns; // cached
         private object[] values; // cached
 
         /// <summary>
@@ -124,9 +126,9 @@ public DataTable GetSchemaTable()
         {
             var schema = GetSchema();
             var schemaTable = GetEmptySchemaDataTable(schema);
-            for (int index = 0; index != schema.ColumnDefinitions.Count; ++index)
+            for (int index = 0, count = columns.Count; index != count; ++index)
             {
-                var column = schema.ColumnDefinitions[index];
+                var column = columns[index];
                 object[] values = new object[]
                 {
                     true,  // AllowDBNull
@@ -165,7 +167,7 @@ private static DataTable GetEmptySchemaDataTable(ISchema schema)
         {
             DataTable schemaTable = new DataTable();
             schemaTable.Locale = CultureInfo.InvariantCulture;
-            schemaTable.MinimumCapacity = schema.ColumnDefinitions.Count;
+            schemaTable.MinimumCapacity = schema.ColumnDefinitions.PhysicalCount;
             schemaTable.Columns.AddRange(new[]
             {
                 new DataColumn(SchemaTableColumn.AllowDBNull, typeof(Boolean)),
@@ -229,7 +231,12 @@ public bool Read()
         /// <summary>
         /// Gets the number of fields in the current record.
         /// </summary>
-        public int FieldCount => GetSchema().ColumnDefinitions.Count;
+        public int FieldCount {
+            get {
+                GetSchema();
+                return columns.Count;
+            }
+        }
 
         /// <summary>
         /// Gets the boolean value from the current record at the given index.
@@ -322,8 +329,8 @@ IDataReader IDataRecord.GetData(int i)
         /// <returns>The type name.</returns>
         public string GetDataTypeName(int i)
         {
-            var schema = GetSchema();
-            return schema.ColumnDefinitions[i].ColumnType.Name;
+            GetSchema();
+            return columns[i].ColumnType.Name;
         }
 
         /// <summary>
@@ -377,8 +384,8 @@ public double GetDouble(int i)
         /// <returns>The type of the value at the given index.</returns>
         public Type GetFieldType(int i)
         {
-            var schema = GetSchema();
-            return schema.ColumnDefinitions[i].ColumnType;
+            GetSchema();
+            return columns[i].ColumnType;
         }
 
         /// <summary>
@@ -443,8 +450,8 @@ public long GetInt64(int i)
         /// <returns>The name of the column at the given index.</returns>
         public string GetName(int i)
         {
-            var schema = GetSchema();
-            return schema.ColumnDefinitions[i].ColumnName;
+            GetSchema();
+            return columns[i].ColumnName;
         }
 
         /// <summary>
@@ -454,8 +461,8 @@ public string GetName(int i)
         /// <returns>The index of the column with the given name.</returns>
         public int GetOrdinal(string name)
         {
-            var schema = GetSchema();
-            return schema.GetOrdinal(name);
+            GetSchema();
+            return columns.GetOrdinal(name);
         }
 
         /// <summary>
@@ -609,6 +616,14 @@ private ISchema GetSchema()
             if (schema == null)
             {
                 schema = Reader.GetSchema();
+                columns = new ColumnCollection();
+                foreach (ColumnDefinition column in schema.ColumnDefinitions)
+                {
+                    if (!column.IsIgnored)
+                    {
+                        columns.AddColumn(column);
+                    }
+                }
             }
             return schema;
         }

FlatFiles/FlatFiles.csproj

@@ -10,17 +10,17 @@
     <RepositoryUrl>https://github.com/jehugaleahsa/FlatFiles.git</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
     <PackageTags>csv;comma;tab;separated;value;delimited;flat;file;fixed;width;fixed-width;length;fixed-length;parser;parsing;parse</PackageTags>
-    <PackageReleaseNotes>Allow handling unrecognized rows when using schema selectors.</PackageReleaseNotes>
+    <PackageReleaseNotes>FlatFileDataReader not correctly ignoring ignored columns in several places.</PackageReleaseNotes>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>FlatFiles.snk</AssemblyOriginatorKeyFile>
-    <Version>4.11.0</Version>
+    <Version>4.12.0</Version>
   </PropertyGroup>
 
   <PropertyGroup>
     <LangVersion>8.0</LangVersion>
     <PackageIconUrl></PackageIconUrl>
-    <AssemblyVersion>4.11.0.0</AssemblyVersion>
-    <FileVersion>4.11.0.0</FileVersion>
+    <AssemblyVersion>4.12.0.0</AssemblyVersion>
+    <FileVersion>4.12.0.0</FileVersion>
     <PackageLicenseFile>UNLICENSE.txt</PackageLicenseFile>
     <PackageIcon>icon.png</PackageIcon>
   </PropertyGroup>