From a81273fc43e96940335ec8f8297dfd7b9ca99b0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Priit=20L=C3=A4tt?= <lattpriit@gmail.com>
Date: Tue, 15 Oct 2024 17:29:08 +0300
Subject: [PATCH] Use errors instead of failures in JUnit reports

---
 src/codemagic/models/junit/definitions.py         |  4 +++-
 src/codemagic/models/xctests/converter.py         |  4 ++--
 src/codemagic/tools/xcode_project.py              | 15 ++++++++-------
 .../xctests/converter/test_xcode_16_converter.py  | 12 ++++++------
 4 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/codemagic/models/junit/definitions.py b/src/codemagic/models/junit/definitions.py
index 2299f07c..48bba132 100644
--- a/src/codemagic/models/junit/definitions.py
+++ b/src/codemagic/models/junit/definitions.py
@@ -37,8 +37,10 @@ def errors(self) -> Optional[int]:
         return sum(suite.errors for suite in self.test_suites if suite.errors)
 
     @property
-    def failures(self) -> int:
+    def failures(self) -> Optional[int]:
         """Total number of failed tests from all testsuites."""
+        if all(suite.failures is None for suite in self.test_suites):
+            return None
         return sum(suite.failures for suite in self.test_suites if suite.failures)
 
     @property
diff --git a/src/codemagic/models/xctests/converter.py b/src/codemagic/models/xctests/converter.py
index c7d15f9e..a9a44da1 100644
--- a/src/codemagic/models/xctests/converter.py
+++ b/src/codemagic/models/xctests/converter.py
@@ -272,8 +272,8 @@ def _get_test_suite(cls, xc_test_suite: XcTestSuite, xc_test_result_summary: XcT
             name=cls._get_test_suite_name(xc_test_suite),
             tests=len(xc_test_suite.test_cases),
             disabled=sum(xc_test_case.is_disabled() for xc_test_case in xc_test_suite.test_cases),
-            errors=None,  # Xcode doesn't differentiate errors from failures
-            failures=sum(xc_test_case.is_failed() for xc_test_case in xc_test_suite.test_cases),
+            errors=sum(xc_test_case.is_failed() for xc_test_case in xc_test_suite.test_cases),
+            failures=None,  # Xcode doesn't differentiate errors from failures, consider everything as error
             package=xc_test_suite.name,
             skipped=sum(xc_test_case.is_skipped() for xc_test_case in xc_test_suite.test_cases),
             time=sum(xc_test_case.get_duration() for xc_test_case in xc_test_suite.test_cases),
diff --git a/src/codemagic/tools/xcode_project.py b/src/codemagic/tools/xcode_project.py
index 652f1fe3..b8915046 100644
--- a/src/codemagic/tools/xcode_project.py
+++ b/src/codemagic/tools/xcode_project.py
@@ -512,13 +512,14 @@ def run_test(
             test_suites = None
             self.logger.error(Colors.RED(f"Parsing test results failed\n{e}\n{e.stderr}"))
         else:
-            if test_suites.errors is not None:
-                failures_message = f"{test_suites.failures} failures and {test_suites.errors} errors"
-            else:
-                failures_message = f"{test_suites.failures} failures"
-
-            message = f"Executed {test_suites.tests} tests with {failures_message} in {test_suites.time:.2f} seconds.\n"
-            self.echo(Colors.BLUE(message))
+            message = f"Executed {test_suites.tests} tests"
+            if test_suites.errors is not None and test_suites.failures is not None:
+                message = f"{message} with {test_suites.failures} failures and {test_suites.errors} errors"
+            elif test_suites.errors is not None:
+                message = f"{message} with {test_suites.errors} errors"
+            elif test_suites.failures is not None:
+                message = f"{message} with {test_suites.errors} failures"
+            self.echo(Colors.BLUE(f"{message} in {test_suites.time:.2f} seconds.\n"))
             TestSuitePrinter(self.echo).print_test_suites(test_suites)
             self._save_test_suite(xcresult, test_suites, output_dir, output_extension)
 
diff --git a/tests/models/xctests/converter/test_xcode_16_converter.py b/tests/models/xctests/converter/test_xcode_16_converter.py
index 9b91acb2..128cfc81 100644
--- a/tests/models/xctests/converter/test_xcode_16_converter.py
+++ b/tests/models/xctests/converter/test_xcode_16_converter.py
@@ -240,8 +240,8 @@ def test_converter(mock_datetime, expected_properties):
 
     assert test_suites.name == "Test - banaan"
     assert test_suites.disabled == 0
-    assert test_suites.errors is None
-    assert test_suites.failures == 3
+    assert test_suites.errors == 3
+    assert test_suites.failures is None
     assert test_suites.skipped == 1
     assert test_suites.tests == 8
     assert test_suites.time == 5.46875
@@ -249,8 +249,8 @@ def test_converter(mock_datetime, expected_properties):
 
     ts = test_suites.test_suites[0]  # Unit tests testsuite assertions
     assert ts.disabled == 0
-    assert ts.errors is None
-    assert ts.failures == 2
+    assert ts.errors == 2
+    assert ts.failures is None
     assert ts.name == "banaanTests [iOS 18.0 iPhone SE (3rd generation)]"
     assert ts.package == "banaanTests"
     assert ts.skipped == 1
@@ -301,8 +301,8 @@ def test_converter(mock_datetime, expected_properties):
 
     ts = test_suites.test_suites[1]  # UI tests testsuite assertions
     assert ts.disabled == 0
-    assert ts.errors is None
-    assert ts.failures == 1
+    assert ts.errors == 1
+    assert ts.failures is None
     assert ts.name == "banaanUITests [iOS 18.0 iPhone SE (3rd generation)]"
     assert ts.package == "banaanUITests"
     assert ts.skipped == 0