Fix example matching for VSTest (#528)

* Fix missing SpecFlow package * Add matching of VSTest _Variant results. Implements the VsTestScenarioOutlineExampleMatcher to detect _Variant style examples. These occur when there are examples which contain characters that are illegal for method names, or when examples contain duplicate values in the first column. Fixes #507
picklesdoc · Jun 15, 2018 · 09b8f97 · 09b8f97
1 parent 47e7989
commit 09b8f97
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 5 deletions.
diff --git a/....DocumentationBuilders.Json.UnitTests/Pickles.DocumentationBuilders.Json.UnitTests.csproj b/....DocumentationBuilders.Json.UnitTests/Pickles.DocumentationBuilders.Json.UnitTests.csproj
@@ -36,6 +36,9 @@
     <Reference Include="Autofac, Version=4.8.0.0, Culture=neutral, PublicKeyToken=17863af14b0044da, processorArchitecture=MSIL">
       <HintPath>..\packages\Autofac.4.8.0\lib\net45\Autofac.dll</HintPath>
     </Reference>
+    <Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
+    </Reference>
     <Reference Include="NFluent, Version=2.2.0.125, Culture=neutral, PublicKeyToken=18828b37b84b1437, processorArchitecture=MSIL">
       <HintPath>..\packages\NFluent.2.2.0\lib\net45\NFluent.dll</HintPath>
     </Reference>
@@ -51,14 +54,17 @@
     <Reference Include="System.IO.Compression.FileSystem" />
     <Reference Include="System.Numerics" />
     <Reference Include="System.Runtime.Serialization" />
+    <Reference Include="System.ValueTuple, Version=4.0.1.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.ValueTuple.4.3.0\lib\netstandard1.0\System.ValueTuple.dll</HintPath>
+    </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System.Data" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
-    <Reference Include="TechTalk.SpecFlow">
-      <HintPath>..\packages\SpecFlow.2.1.0\lib\net45\TechTalk.SpecFlow.dll</HintPath>
+    <Reference Include="TechTalk.SpecFlow, Version=2.3.2.0, Culture=neutral, PublicKeyToken=0778194805d6db41, processorArchitecture=MSIL">
+      <HintPath>..\packages\SpecFlow.2.3.2\lib\net45\TechTalk.SpecFlow.dll</HintPath>
     </Reference>
     <Reference Include="TestingHelpers, Version=2.1.0.178, Culture=neutral, PublicKeyToken=96bf224d23c43e59, processorArchitecture=MSIL">
       <HintPath>..\packages\System.IO.Abstractions.TestingHelpers.2.1.0.178\lib\net40\TestingHelpers.dll</HintPath>
@@ -111,7 +117,9 @@
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
     <Error Condition="!Exists('..\packages\NUnit.3.10.1\build\NUnit.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\NUnit.3.10.1\build\NUnit.props'))" />
+    <Error Condition="!Exists('..\packages\SpecFlow.2.3.2\build\SpecFlow.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\SpecFlow.2.3.2\build\SpecFlow.targets'))" />
   </Target>
+  <Import Project="..\packages\SpecFlow.2.3.2\build\SpecFlow.targets" Condition="Exists('..\packages\SpecFlow.2.3.2\build\SpecFlow.targets')" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">

diff --git a/src/Pickles/Pickles.DocumentationBuilders.Json.UnitTests/app.config b/src/Pickles/Pickles.DocumentationBuilders.Json.UnitTests/app.config
@@ -1,5 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
+  <configSections>
+    <section name="specFlow" type="TechTalk.SpecFlow.Configuration.ConfigurationSectionHandler, TechTalk.SpecFlow" />
+  </configSections>
   <runtime>
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
       <dependentAssembly>
@@ -20,4 +23,6 @@
       </dependentAssembly>
     </assemblyBinding>
   </runtime>
-</configuration>
+<specFlow>
+    <!-- For additional details on SpecFlow configuration options see http://go.specflow.org/doc-config -->
+  </specFlow></configuration>
diff --git a/src/Pickles/Pickles.DocumentationBuilders.Json.UnitTests/packages.config b/src/Pickles/Pickles.DocumentationBuilders.Json.UnitTests/packages.config
@@ -1,8 +1,11 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Autofac" version="4.8.0" targetFramework="net45" />
+  <package id="Newtonsoft.Json" version="9.0.1" targetFramework="net45" />
   <package id="NFluent" version="2.2.0" targetFramework="net45" />
   <package id="NUnit" version="3.10.1" targetFramework="net45" />
+  <package id="SpecFlow" version="2.3.2" targetFramework="net45" />
   <package id="System.IO.Abstractions" version="2.1.0.178" targetFramework="net45" />
   <package id="System.IO.Abstractions.TestingHelpers" version="2.1.0.178" targetFramework="net45" />
+  <package id="System.ValueTuple" version="4.3.0" targetFramework="net45" />
 </packages>
diff --git a/src/Pickles/Pickles.TestFrameworks.UnitTests/StandardTestSuiteForScenarioOutlines.cs b/src/Pickles/Pickles.TestFrameworks.UnitTests/StandardTestSuiteForScenarioOutlines.cs
@@ -19,6 +19,7 @@
 //  --------------------------------------------------------------------------------------------------------------------
 
 
+using System.Collections.Generic;
 using NFluent;
 
 
@@ -136,7 +137,30 @@ public void ThenCanReadExamplesWithRegexValuesFromScenarioOutline_ShouldBeTestRe
 
             var feature = new Feature { Name = "Scenarios With Special Characters" };
 
-            var scenarioOutline = new ScenarioOutline { Name = "This scenario contains examples with Regex-special characters", Feature = feature };
+            var scenarioOutline = new ScenarioOutline
+            {
+                Name = "This scenario contains examples with Regex-special characters",
+                Feature = feature,
+                Examples = new List<Example>
+                {
+                    new Example
+                    {
+                        TableArgument = new ExampleTable
+                        {
+                            DataRows = new List<TableRow>
+                            {
+                                new TableRow("**"),
+                                new TableRow("++"),
+                                new TableRow(".*"),
+                                new TableRow("[]"),
+                                new TableRow("{}"),
+                                new TableRow("()"),
+                                new TableRow(@"^.*(?<foo>BAR)\s[^0-9]{3,4}A+$"),
+                            }
+                        }
+                    }
+                }
+            };
 
             TestResult exampleResultOutline = results.GetScenarioOutlineResult(scenarioOutline);
             Check.That(exampleResultOutline).IsEqualTo(TestResult.Passed);

diff --git a/...kles.TestFrameworks.UnitTests/VsTest/WhenParsingVsTestResultsFileWithIndividualResults.cs b/...kles.TestFrameworks.UnitTests/VsTest/WhenParsingVsTestResultsFileWithIndividualResults.cs
@@ -57,6 +57,12 @@ public WhenParsingVsTestResultsFileWithIndividualResults()
             base.ThenCanReadIndividualResultsFromScenarioOutline_MultipleExampleSections_ShouldBeTestResultFailed();
         }
 
+        [Test]
+        public new void ThenCanReadExamplesWithRegexValuesFromScenarioOutline_ShouldBeTestResultPassed()
+        {
+            base.ThenCanReadExamplesWithRegexValuesFromScenarioOutline_ShouldBeTestResultPassed();
+        }
+
         [Test]
         public new void ThenCanReadExamplesWithLongExampleValues()
         {

diff --git a/src/Pickles/Pickles.TestFrameworks/VsTest/VsTestScenarioOutlineExampleMatcher.cs b/src/Pickles/Pickles.TestFrameworks/VsTest/VsTestScenarioOutlineExampleMatcher.cs
@@ -18,6 +18,8 @@
 //  </copyright>
 //  --------------------------------------------------------------------------------------------------------------------
 
+using System.Linq;
+using System.Text.RegularExpressions;
 using System.Xml.Linq;
 
 using PicklesDoc.Pickles.ObjectModel;
@@ -26,6 +28,9 @@ namespace PicklesDoc.Pickles.TestFrameworks.VsTest
 {
     public class VsTestScenarioOutlineExampleMatcher : IScenarioOutlineExampleMatcher
     {
+        private static readonly Regex VariantRegex = new Regex(@"(.*)_Variant([\d*])", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+        private const int VariantNumberGroup = 2;
+
         public bool IsMatch(ScenarioOutline scenarioOutline, string[] exampleValues, object scenarioElement)
         {
             var element = (XElement)scenarioElement;
@@ -51,7 +56,23 @@ public bool IsMatch(ScenarioOutline scenarioOutline, string[] exampleValues, obj
                 .Replace('Ø', 'O')
                 .Replace('Å', 'A');
 
-      var isMatch = element.Name().ToUpperInvariant()
+            var variantMatch = VariantRegex.Match(element.Name().ToUpperInvariant());
+            if (variantMatch.Success)
+            {
+                int variantNumber;
+                if (int.TryParse(variantMatch.Groups[VariantNumberGroup].Value, out variantNumber))
+                {
+                    if (scenarioOutline.Examples?.Count > 0)
+                    {
+                        var allExamples = scenarioOutline.Examples.SelectMany(x => x.TableArgument.DataRows);
+                        var example = allExamples.ElementAt(variantNumber);
+
+                        return example.Cells.SequenceEqual(exampleValues);
+                    }
+                }
+            }
+
+            var isMatch = element.Name().ToUpperInvariant()
                 .EndsWith(matchValue);
 
             return isMatch;