Fix trx in case of exit code != 0

src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxDataConsumer.cs

@@ -38,6 +38,7 @@ internal sealed class TrxReportGenerator :
     private readonly IOutputDevice _outputDisplay;
     private readonly ITestFramework _testFramework;
     private readonly ITestFrameworkCapabilities _testFrameworkCapabilities;
+    private readonly ITestApplicationProcessExitCode _testApplicationProcessExitCode;
     private readonly TrxTestApplicationLifecycleCallbacks? _trxTestApplicationLifecycleCallbacks;
     private readonly ILogger<TrxReportGenerator> _logger;
     private readonly List<TestNodeUpdateMessage> _tests = [];
@@ -62,6 +63,7 @@ public TrxReportGenerator(
         IOutputDevice outputDisplay,
         ITestFramework testFramework,
         ITestFrameworkCapabilities testFrameworkCapabilities,
+        ITestApplicationProcessExitCode testApplicationProcessExitCode,
         // Can be null in case of server mode
         TrxTestApplicationLifecycleCallbacks? trxTestApplicationLifecycleCallbacks,
         ILogger<TrxReportGenerator> logger)
@@ -75,6 +77,7 @@ public TrxReportGenerator(
         _outputDisplay = outputDisplay;
         _testFramework = testFramework;
         _testFrameworkCapabilities = testFrameworkCapabilities;
+        _testApplicationProcessExitCode = testApplicationProcessExitCode;
         _trxTestApplicationLifecycleCallbacks = trxTestApplicationLifecycleCallbacks;
         _logger = logger;
         _isEnabled = commandLineOptionsService.IsOptionSet(TrxReportGeneratorCommandLine.TrxReportOptionName);
@@ -232,16 +235,15 @@ public async Task OnTestSessionFinishingAsync(SessionUid sessionUid, Cancellatio
 
             ApplicationStateGuard.Ensure(_testStartTime is not null);
 
+            int exitCode = _testApplicationProcessExitCode.GetProcessExitCode();
             TrxReportEngine trxReportGeneratorEngine = new(_testApplicationModuleInfo, _environment, _commandLineOptionsService, _configuration,
             _clock, _tests.ToArray(), _failedTestsCount, _passedTestsCount, _notExecutedTestsCount, _timeoutTestsCount, _artifactsByExtension, _artifactsByTestNode,
-            _adapterSupportTrxCapability, _testFramework, _testStartTime.Value, cancellationToken);
+            _adapterSupportTrxCapability, _testFramework, _testStartTime.Value, exitCode, cancellationToken);
             string reportFileName = await trxReportGeneratorEngine.GenerateReportAsync();
 
             if (
-
                 // TestController is not used when we run in server mode
                 _commandLineOptionsService.IsOptionSet(PlatformCommandLineProvider.ServerOptionKey) ||
-
                 // If crash dump is not enabled we run trx in-process only
                 !_commandLineOptionsService.IsOptionSet(CrashDumpCommandLineOptions.CrashDumpOptionName))
             {

src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxProcessLifetimeHandler.cs

@@ -164,6 +164,7 @@ public async Task OnTestHostProcessExitedAsync(ITestHostProcessInformation testH
                 adapterSupportTrxCapability: null,
                 new TestAdapterInfo(_testAdapterInformationRequest!.TestAdapterId, _testAdapterInformationRequest.TestAdapterVersion),
                 _startTime,
+                testHostProcessInformation.ExitCode,
                 cancellation);
 
             await _messageBus.PublishAsync(
@@ -194,6 +195,7 @@ await _messageBus.PublishAsync(
                false,
                new TestAdapterInfo(_testAdapterInformationRequest!.TestAdapterId, _testAdapterInformationRequest.TestAdapterVersion),
                _startTime,
+               testHostProcessInformation.ExitCode,
                cancellation);
 
             await trxReportGeneratorEngine.AddArtifactsAsync(trxFile, artifacts);

src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxReportEngine.cs

@@ -102,11 +102,12 @@ internal sealed partial class TrxReportEngine
     private readonly ITestFramework _testFrameworkAdapter;
     private readonly DateTimeOffset _testStartTime;
     private readonly CancellationToken _cancellationToken;
+    private readonly int _exitCode;
     private readonly XNamespace _namespaceUri = XNamespace.Get("http://microsoft.com/schemas/VisualStudio/TeamTest/2010");
     private readonly IFileSystem _fileSystem;
     private readonly bool _isCopyingFileAllowed;
 
-    public TrxReportEngine(ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, int notExecutedTestsCount, int timeoutTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, CancellationToken cancellationToken)
+    public TrxReportEngine(ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, int notExecutedTestsCount, int timeoutTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, int exitCode, CancellationToken cancellationToken)
     : this(
         new SystemFileSystem(),
         testApplicationModuleInfo,
@@ -124,11 +125,12 @@ public TrxReportEngine(ITestApplicationModuleInfo testApplicationModuleInfo, IEn
         adapterSupportTrxCapability,
         testFrameworkAdapter,
         testStartTime,
+        exitCode,
         cancellationToken)
     {
     }
 
-    public TrxReportEngine(IFileSystem fileSystem, ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, int notExecutedTestsCount, int timeoutTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, CancellationToken cancellationToken, bool isCopyingFileAllowed = true)
+    public TrxReportEngine(IFileSystem fileSystem, ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, int notExecutedTestsCount, int timeoutTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, int exitCode, CancellationToken cancellationToken, bool isCopyingFileAllowed = true)
     {
         _testApplicationModuleInfo = testApplicationModuleInfo;
         _environment = environment;
@@ -146,6 +148,7 @@ public TrxReportEngine(IFileSystem fileSystem, ITestApplicationModuleInfo testAp
         _testFrameworkAdapter = testFrameworkAdapter;
         _testStartTime = testStartTime;
         _cancellationToken = cancellationToken;
+        _exitCode = exitCode;
         _fileSystem = fileSystem;
         _isCopyingFileAllowed = isCopyingFileAllowed;
     }
@@ -175,9 +178,9 @@ public async Task<string> GenerateReportAsync(string testHostCrashInfo = "", boo
             AddTestLists(testRun, uncategorizedTestId);
 
             // NotExecuted is the status for the skipped test.
-            resultSummaryOutcome = isTestHostCrashed ? "Failed" : resultSummaryOutcome is "Passed" or "NotExecuted" ? "Completed" : resultSummaryOutcome;
+            resultSummaryOutcome = isTestHostCrashed || _exitCode != ExitCodes.Success ? "Failed" : resultSummaryOutcome is "Passed" or "NotExecuted" ? "Completed" : resultSummaryOutcome;
 
-            await AddResultSummaryAsync(testRun, resultSummaryOutcome, runDeploymentRoot, testHostCrashInfo, isTestHostCrashed);
+            await AddResultSummaryAsync(testRun, resultSummaryOutcome, runDeploymentRoot, testHostCrashInfo, _exitCode, isTestHostCrashed);
 
             // will need catch Unauthorized access
             document.Add(testRun);
@@ -289,7 +292,7 @@ private async Task AddArtifactsToCollectionAsync(Dictionary<IExtension, List<Ses
         }
     }
 
-    private async Task AddResultSummaryAsync(XElement testRun, string resultSummaryOutcome, string runDeploymentRoot, string testHostCrashInfo, bool isTestHostCrashed = false)
+    private async Task AddResultSummaryAsync(XElement testRun, string resultSummaryOutcome, string runDeploymentRoot, string testHostCrashInfo, int exitCode, bool isTestHostCrashed = false)
     {
         var resultSummary = new XElement(
             _namespaceUri + "ResultSummary",
@@ -329,6 +332,19 @@ private async Task AddResultSummaryAsync(XElement testRun, string resultSummaryO
             runInfo.Add(text);
             runInfos.Add(runInfo);
         }
+        else if (exitCode != ExitCodes.Success)
+        {
+            var runInfos = new XElement(_namespaceUri + "RunInfos");
+            resultSummary.Add(runInfos);
+            var runInfo = new XElement(
+                _namespaceUri + "RunInfo",
+                new XAttribute("computerName", _environment.MachineName),
+                new XAttribute("outcome", "Error"),
+                new XAttribute("timestamp", _clock.UtcNow.DateTime));
+            var text = new XElement(_namespaceUri + "Text", $"Exit code indicates failure: '{exitCode}'. Please refer to https://aka.ms/testingplatform/exitcodes for more information.");
+            runInfo.Add(text);
+            runInfos.Add(runInfo);
+        }
 
         if (_artifactsByExtension.Count == 0)
         {

src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxReportExtensions.cs

@@ -36,6 +36,7 @@ public static void AddTrxReportProvider(this ITestApplicationBuilder builder)
                 serviceProvider.GetOutputDevice(),
                 serviceProvider.GetTestFramework(),
                 serviceProvider.GetTestFrameworkCapabilities(),
+                serviceProvider.GetTestApplicationProcessExitCode(),
                 serviceProvider.GetService<TrxTestApplicationLifecycleCallbacks>(),
                 serviceProvider.GetLoggerFactory().CreateLogger<TrxReportGenerator>()));
 

src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxTestApplicationLifecycleCallbacks.cs

@@ -24,13 +24,10 @@ public TrxTestApplicationLifecycleCallbacks(
         IEnvironment environment)
     {
         _isEnabled =
-
            // TrxReportGenerator is enabled only when trx report is enabled
            commandLineOptionsService.IsOptionSet(TrxReportGeneratorCommandLine.TrxReportOptionName) &&
-
            // TestController is not used when we run in server mode
            !commandLineOptionsService.IsOptionSet(PlatformCommandLineProvider.ServerOptionKey) &&
-
            // If crash dump is not enabled we run trx in-process only
            commandLineOptionsService.IsOptionSet(CrashDumpCommandLineOptions.CrashDumpOptionName);
 

src/Platform/Microsoft.Testing.Platform/Hosts/ConsoleTestHost.cs

@@ -94,7 +94,7 @@ await ExecuteRequestAsync(
             // Get the exit code service to be able to set the exit code
             ITestApplicationProcessExitCode testApplicationResult = ServiceProvider.GetTestApplicationProcessExitCode();
             statistics = testApplicationResult.GetStatistics();
-            exitCode = await testApplicationResult.GetProcessExitCodeAsync();
+            exitCode = testApplicationResult.GetProcessExitCode();
 
             await _logger.LogInformationAsync($"Test session '{ServiceProvider.GetTestSessionContext().SessionId}' ended with exit code '{exitCode}' in {consoleRunStarted.Elapsed}");
 

src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs

@@ -324,6 +324,10 @@ public async Task<ITestHost> BuildAsync(
             serviceProvider.AddService(nonCooperativeParentProcessListener);
         }
 
+        // Add TestApplicationResultProxy
+        TestApplicationResultProxy testApplicationResultProxy = new();
+        serviceProvider.AddService(testApplicationResultProxy);
+
         // ============= SETUP COMMON SERVICE USED IN ALL MODES END ===============//
 
         // ============= SELECT AND RUN THE ACTUAL MODE ===============//
@@ -730,7 +734,11 @@ private async Task<ITestFramework> BuildTestFrameworkAsync(TestFrameworkBuilderD
             serviceProvider.GetTestApplicationCancellationTokenSource(),
             serviceProvider.GetCommandLineOptions(),
             serviceProvider.GetEnvironment());
-        await RegisterAsServiceOrConsumerOrBothAsync(testApplicationResult, serviceProvider, dataConsumersBuilder);
+
+        // Set the concrete TestApplicationResult
+        TestApplicationResultProxy testApplicationResultProxy = serviceProvider.GetRequiredService<TestApplicationResultProxy>();
+        testApplicationResultProxy.SetTestApplicationProcessExitCode(testApplicationResult);
+        await RegisterAsServiceOrConsumerOrBothAsync(testApplicationResultProxy, serviceProvider, dataConsumersBuilder);
 
         // We register the data consumer handler if we're connected to the dotnet test pipe
         if (pushOnlyProtocolDataConsumer is not null)

src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx

@@ -631,4 +631,7 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is
   <data name="MissingClientPortFoJsonRpc" xml:space="preserve">
     <value>Expected --client-port when jsonRpc protocol is used.</value>
   </data>
-</root>
+  <data name="TestApplicationResultNotReady" xml:space="preserve">
+    <value>The ITestApplicationResult has not been built yet.</value>
+  </data>
+</root>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf

@@ -730,6 +730,11 @@ Platné hodnoty jsou Normal a Detailed. Výchozí hodnota je Normal.</target>
         <target state="translated">Výsledek testovací aplikace</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">Režim VSTest podporuje jen jeden TestApplicationBuilder na proces.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf

@@ -730,6 +730,11 @@ Gültige Werte sind „Normal“, „Detailed“. Der Standardwert ist „Normal
         <target state="translated">Testanwendungsergebnis</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">Der VSTest-Modus unterstützt nur einen einzelnen TestApplicationBuilder pro Prozess.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf

@@ -730,6 +730,11 @@ Los valores válidos son 'Normal', 'Detallado'. El valor predeterminado es 'Norm
         <target state="translated">Resultado de la aplicación de prueba</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">El modo VSTest solo admite un único TestApplicationBuilder por proceso</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf

@@ -730,6 +730,11 @@ Les valeurs valides sont « Normal » et « Détaillé ». La valeur par dé
         <target state="translated">Résultat de l’application de test</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">Le mode VSTest ne prend en charge qu’un seul TestApplicationBuilder par processus</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf

@@ -730,6 +730,11 @@ I valori validi sono 'Normal', 'Detailed'. L'impostazione predefinita è 'Normal
         <target state="translated">Risultato del test dell'applicazione</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">La modalità VSTest supporta solo un singolo TestApplicationBuilder per processo</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf

@@ -731,6 +731,11 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'.</source>
         <target state="translated">テスト アプリケーションの結果</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">VSTest モードは、プロセスごとに 1 つの TestApplicationBuilder のみをサポートします</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf

@@ -730,6 +730,11 @@ Valid values are 'Normal', 'Detailed'. Default is 'Normal'.</source>
         <target state="translated">테스트 애플리케이션 결과</target>
         <note />
       </trans-unit>
+      <trans-unit id="TestApplicationResultNotReady">
+        <source>The ITestApplicationResult has not been built yet.</source>
+        <target state="new">The ITestApplicationResult has not been built yet.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="TestApplicationVSTestModeTooManyBuilders">
         <source>VSTest mode only supports a single TestApplicationBuilder per process</source>
         <target state="translated">VSTest 모드는 프로세스당 단일 TestApplicationBuilder만 지원합니다.</target>