Capture Xcode result bundle if unit tests fail

.buildkite/pipeline.quick.yml

@@ -1,164 +1,108 @@
+env:
+  LANG: "en_GB.UTF-8"
+
 steps:
   - label: macOS 11 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "macOS"
     commands:
-      - make bootstrap
-      - make analyze test
+      - ./scripts/run-unit-tests.sh PLATFORM=macOS
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: macOS 10.14 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-10.14
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "macOS"
     commands:
-      - make bootstrap
-      - make analyze test
+      - ./scripts/run-unit-tests.sh PLATFORM=macOS
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: iOS 13 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "iOS"
-      OS: "13.7"
     concurrency: 3
     concurrency_group: cocoa-unit-tests
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=13.7
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: iOS 12 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "iOS"
-      OS: "12.4"
     concurrency: 3
     concurrency_group: cocoa-unit-tests
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=12.4
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: iOS 11 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "iOS"
-      OS: "11.4"
     concurrency: 3
     concurrency_group: cocoa-unit-tests
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=11.4
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: iOS 10 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-10.15
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "iOS"
-      OS: "10.3.1"
-      DEVICE: "iPhone 5s"
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=10.3.1 DEVICE=iPhone\ 5s
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: tvOS 13 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "tvOS"
-      OS: "13.3"
     concurrency: 3
     concurrency_group: cocoa-unit-tests
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=13.3
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: tvOS 12 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "tvOS"
-      OS: "12.4"
     concurrency: 3
     concurrency_group: cocoa-unit-tests
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=12.4
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: tvOS 11 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "tvOS"
-      OS: "11.4"
     concurrency: 3
     concurrency_group: cocoa-unit-tests
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=11.4
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: tvOS 10 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-10.15
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "tvOS"
-      OS: "10.2"
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=10.2
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: ':apple: macOS 10.15 full end-to-end tests'
     depends_on:

.buildkite/pipeline.yml

@@ -30,8 +30,6 @@ steps:
       queue: opensource-mac-cocoa-10.15
     concurrency: 3
     concurrency_group: cocoa-unit-tests
-    env:
-      LANG: "en_GB.UTF-8"
     commands:
       - mkdir -p features/fixtures/carthage-proj
       - make build_swift
@@ -56,88 +54,55 @@ steps:
       queue: opensource-mac-cocoa-10.15
     concurrency: 3
     concurrency_group: cocoa-unit-tests
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "macOS"
     commands:
-      - make bootstrap
-      - make analyze test
+      - ./scripts/run-unit-tests.sh PLATFORM=macOS
+    artifact_paths:
+      - logs/*
 
   - label: macOS 10.13 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-10.13
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "macOS"
     commands:
-      - make bootstrap
-      - make analyze test
+      - ./scripts/run-unit-tests.sh PLATFORM=macOS
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: iOS 14 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "iOS"
-      OS: "14.3"
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=14.3
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: iOS 9 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-10.13
-    env:
-      LANG: "en_GB.UTF-8"
-      PLATFORM: "iOS"
-      OS: "9.3"
-      DEVICE: "iPhone 5s"
     commands:
-      - make bootstrap
-      - TEST_CONFIGURATION=Debug make test
-      - TEST_CONFIGURATION=Release make test
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=9.3 DEVICE=iPhone\ 5s
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: tvOS 14 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-11
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "tvOS"
-      OS: "14.3"
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=14.3
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: tvOS 9 unit tests
     timeout_in_minutes: 10
     agents:
       queue: opensource-mac-cocoa-10.13
-    env:
-      LANG: "en_GB.UTF-8"
-      TEST_CONFIGURATION: "Debug"
-      PLATFORM: "tvOS"
-      OS: "9.2"
     commands:
-      - make bootstrap
-      - make test
+      - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=9.2
     artifact_paths:
-      - xcodebuild.log
+      - logs/*
 
   - label: ':ios: iOS 14 barebones end-to-end tests'
     depends_on:

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 build/
+logs/
 *.pbxuser
 !default.pbxuser
 *.mode1v3
@@ -10,6 +11,7 @@ build/
 !default.perspectivev3
 xcuserdata
 *.xccheckout
+*.xcresult
 *.moved-aside
 DerivedData
 *.hmap

Makefile

@@ -15,8 +15,9 @@
 PLATFORM?=iOS
 OS?=latest
 TEST_CONFIGURATION?=Debug
+XCODEBUILD_EXTRA_ARGS?=
 DATA_PATH=DerivedData
-BUILD_FLAGS=-project Bugsnag.xcodeproj -scheme Bugsnag-$(PLATFORM) -derivedDataPath $(DATA_PATH)
+BUILD_FLAGS=-project Bugsnag.xcodeproj -scheme Bugsnag-$(PLATFORM) -derivedDataPath $(DATA_PATH) $(XCODEBUILD_EXTRA_ARGS)
 
 ifeq ($(PLATFORM),macOS)
  SDK?=macosx

scripts/run-unit-tests.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -o nounset
+
+xcresult=$(date '+BugsnagTests-%Y-%m-%d-%H-%M-%S.xcresult')
+
+die() {
+	status=$?
+	echo "^^^ +++"
+	mkdir -p logs
+	[[ -f xcodebuild.log ]] && mv xcodebuild.log logs/
+	[[ -d $xcresult ]] && zip -qr "logs/$xcresult.zip" "$xcresult"
+	exit $status
+}
+
+
+echo "--- Analyze"
+
+make analyze "$@" || die
+
+rm -rf DerivedData
+
+
+echo "--- Test"
+
+make test XCODEBUILD_EXTRA_ARGS="-resultBundlePath $xcresult" "$@" || die
+
+rm -rf "$xcresult"