Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a CollectDataForEachTestSeparately setting #510

Merged
merged 2 commits into from
Jun 10, 2018
Merged

Add a CollectDataForEachTestSeparately setting #510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d65aff5
Add setting to run tests one at a time for InProcDataCollectors
drognanar May 22, 2018
b58597c
Handle settings for old LiveUnitTesting collector
drognanar May 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 36 additions & 23 deletions src/NUnitTestAdapter/AdapterSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,16 @@ public interface IAdapterSettings
int? RandomSeed { get; }
bool RandomSeedSpecified { get; }
bool InProcDataCollectorsAvailable { get; }
bool CollectDataForEachTestSeparately { get; }
bool SynchronousEvents { get; }
string DomainUsage { get; }
bool DumpXmlTestDiscovery { get; }
bool DumpXmlTestResults { get; }
string DomainUsage { get; }
bool DumpXmlTestDiscovery { get; }
bool DumpXmlTestResults { get; }

/// <summary>
/// Syntax documentation <see cref="https://github.com/nunit/docs/wiki/Template-Based-Test-Naming"/>
/// </summary>
string DefaultTestNamePattern { get; }
string DefaultTestNamePattern { get; }

void Load(IDiscoveryContext context);
void Load(string settingsXml);
Expand Down Expand Up @@ -143,6 +144,8 @@ public AdapterSettings(TestLogger logger)
public int? RandomSeed { get; private set; }
public bool RandomSeedSpecified { get; private set; }

public bool CollectDataForEachTestSeparately { get; private set; }

public bool InProcDataCollectorsAvailable { get; private set; }

public bool SynchronousEvents { get; private set; }
Expand Down Expand Up @@ -195,6 +198,7 @@ public void Load(string settingsXml)
DisableAppDomain = GetInnerTextAsBool(runConfiguration, nameof(DisableAppDomain), false);
DisableParallelization = GetInnerTextAsBool(runConfiguration, nameof(DisableParallelization), false);
DesignMode = GetInnerTextAsBool(runConfiguration, nameof(DesignMode), false);
CollectDataForEachTestSeparately = GetInnerTextAsBool(runConfiguration, nameof(CollectDataForEachTestSeparately), false);

TestProperties = new Dictionary<string, string>();
foreach (XmlNode node in doc.SelectNodes("RunSettings/TestRunParameters/Parameter"))
Expand All @@ -205,7 +209,7 @@ public void Load(string settingsXml)
TestProperties.Add(key, value);
}

// NUnit settings
// NUnit settings
InternalTraceLevel = GetInnerTextWithLog(nunitNode, nameof(InternalTraceLevel), "Off", "Error", "Warning", "Info", "Verbose", "Debug");
WorkDirectory = GetInnerTextWithLog(nunitNode, nameof(WorkDirectory));
DefaultTimeout = GetInnerTextAsInt(nunitNode, nameof(DefaultTimeout), 0);
Expand All @@ -220,8 +224,8 @@ public void Load(string settingsXml)
RandomSeed = new Random().Next();
DefaultTestNamePattern = GetInnerTextWithLog(nunitNode, nameof(DefaultTestNamePattern));

DumpXmlTestDiscovery = GetInnerTextAsBool(nunitNode, nameof(DumpXmlTestDiscovery),false);
DumpXmlTestResults= GetInnerTextAsBool(nunitNode, nameof(DumpXmlTestResults), false);
DumpXmlTestDiscovery = GetInnerTextAsBool(nunitNode, nameof(DumpXmlTestDiscovery), false);
DumpXmlTestResults = GetInnerTextAsBool(nunitNode, nameof(DumpXmlTestResults), false);



Expand All @@ -241,22 +245,31 @@ public void Load(string settingsXml)
Verbosity = 1;
#endif

// If any in proc data collector will be instantiated by the TestPlatform run tests sequentially.
var inProcDataCollectorNode = doc.SelectSingleNode("RunSettings/InProcDataCollectionRunSettings/InProcDataCollectors");
InProcDataCollectorsAvailable = inProcDataCollectorNode != null && inProcDataCollectorNode.SelectNodes("InProcDataCollector").Count > 0;
if (InProcDataCollectorsAvailable)

// Older versions of VS do not pass the CollectDataForEachTestSeparately configuration together with the LiveUnitTesting collector.
// However, the adapter is expected to run in CollectDataForEachTestSeparately mode.
// As a result for backwards compatibility reasons enable CollectDataForEachTestSeparately mode whenever LiveUnitTesting collector is being used.
var hasLiveUnitTestingDataCollector = inProcDataCollectorNode?.SelectSingleNode("InProcDataCollector[@uri='InProcDataCollector://Microsoft/LiveUnitTesting/1.0']") != null;

// TestPlatform can opt-in to run tests one at a time so that the InProcDataCollectors can collect the data for each one of them separately.
// In that case, we need to ensure that tests do not run in parallel and the test started/test ended events are sent synchronously.
if (CollectDataForEachTestSeparately || hasLiveUnitTestingDataCollector)
{
NumberOfTestWorkers = 0;
DomainUsage = "None";
SynchronousEvents = true;
if (Verbosity >= 4)
{
_logger.Info($"InProcDataCollectors are available: turning off Parallel, DomainUsage=None, SynchronousEvents=true");
if (!InProcDataCollectorsAvailable)
{
_logger.Info("CollectDataForEachTestSeparately is set, which is used to make InProcDataCollectors collect data for each test separately. No InProcDataCollectors can be found, thus the tests will run slower unnecessarily.");
}
}
}

// If DisableAppDomain settings is passed from the testplatform, set the DomainUsage to None.
if(DisableAppDomain)
if (DisableAppDomain)
{
DomainUsage = "None";
}
Expand Down Expand Up @@ -301,13 +314,13 @@ public void RestoreRandomSeed(string dirname)
private void UpdateNumberOfTestWorkers()
{
// Overriding the NumberOfTestWorkers if DisableParallelization is true.
if(DisableParallelization && NumberOfTestWorkers < 0)
if (DisableParallelization && NumberOfTestWorkers < 0)
{
NumberOfTestWorkers = 0;
}
else if(DisableParallelization && NumberOfTestWorkers > 0)
else if (DisableParallelization && NumberOfTestWorkers > 0)
{
if(_logger.Verbosity > 0)
if (_logger.Verbosity > 0)
{
_logger.Warning(string.Format("DisableParallelization:{0} & NumberOfTestWorkers:{1} are conflicting settings, hence not running in parallel", DisableParallelization, NumberOfTestWorkers));
}
Expand Down Expand Up @@ -339,42 +352,42 @@ private string GetInnerText(XmlNode startNode, string xpath, bool log, params st
"Invalid value {0} passed for element {1}.", val, xpath));
}


}
if (log)
Log(xpath,val);
Log(xpath, val);

return val;
}

private int GetInnerTextAsInt(XmlNode startNode, string xpath, int defaultValue)
{
var temp = GetInnerTextAsNullableInt(startNode, xpath,false);
var temp = GetInnerTextAsNullableInt(startNode, xpath, false);
var res = defaultValue;
if (temp != null)
res = temp.Value;
Log(xpath, res);
return res;
}

private int? GetInnerTextAsNullableInt(XmlNode startNode, string xpath,bool log=true)
private int? GetInnerTextAsNullableInt(XmlNode startNode, string xpath, bool log = true)
{
string temp = GetInnerText(startNode, xpath,log);
string temp = GetInnerText(startNode, xpath, log);
int? res = null;
if (!string.IsNullOrEmpty(temp))
res = int.Parse(temp);
if (log)
Log(xpath,res);
Log(xpath, res);
return res;
}

private bool GetInnerTextAsBool(XmlNode startNode, string xpath, bool defaultValue)
{
string temp = GetInnerText(startNode, xpath,false);
string temp = GetInnerText(startNode, xpath, false);
bool res = defaultValue;
if (!string.IsNullOrEmpty(temp))
res = bool.Parse(temp);
Log(xpath,res);
Log(xpath, res);
return res;
}

Expand Down
41 changes: 40 additions & 1 deletion src/NUnitTestAdapterTests/AdapterSettingsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,27 @@ public void DefaultTestNamePattern()
Assert.That(_settings.DefaultTestNamePattern,Is.EqualTo("{m}{a:1000}"));
}

[Test]
public void CollectDataForEachTestSeparately()
{
_settings.Load(@"
<RunSettings>
<RunConfiguration>
<CollectDataForEachTestSeparately>true</CollectDataForEachTestSeparately>
</RunConfiguration>
<InProcDataCollectionRunSettings>
<InProcDataCollectors>
<InProcDataCollector friendlyName='DummyCollectorName' uri='InProcDataCollector://NUnit/DummyCollectorName' />
</InProcDataCollectors>
</InProcDataCollectionRunSettings>
</RunSettings>");

Assert.Null(_settings.DomainUsage);
Assert.True(_settings.SynchronousEvents);
Assert.That(_settings.NumberOfTestWorkers, Is.Zero);
Assert.True(_settings.InProcDataCollectorsAvailable);
}

[Test]
public void InProcDataCollector()
{
Expand All @@ -254,7 +275,25 @@ public void InProcDataCollector()
</InProcDataCollectionRunSettings>
</RunSettings>");

Assert.That(_settings.DomainUsage, Is.EqualTo("None"));
Assert.Null(_settings.DomainUsage);
Assert.False(_settings.SynchronousEvents);
Assert.That(_settings.NumberOfTestWorkers, Is.EqualTo(-1));
Assert.True(_settings.InProcDataCollectorsAvailable);
}

[Test]
public void LiveUnitTestingDataCollector()
{
_settings.Load(@"
<RunSettings>
<InProcDataCollectionRunSettings>
<InProcDataCollectors>
<InProcDataCollector friendlyName='DummyCollectorName' uri='InProcDataCollector://Microsoft/LiveUnitTesting/1.0' />
</InProcDataCollectors>
</InProcDataCollectionRunSettings>
</RunSettings>");

Assert.Null(_settings.DomainUsage);
Assert.True(_settings.SynchronousEvents);
Assert.That(_settings.NumberOfTestWorkers, Is.Zero);
Assert.True(_settings.InProcDataCollectorsAvailable);
Expand Down