From 39f61f159e105eade0efa629c2ab5e6db9936eb8 Mon Sep 17 00:00:00 2001 From: Nick Dowell Date: Fri, 21 May 2021 10:07:20 +0100 Subject: [PATCH] Capture Xcode result bundle if unit tests fail --- .buildkite/pipeline.quick.yml | 102 ++++++++-------------------------- .buildkite/pipeline.yml | 61 +++++--------------- .gitignore | 2 + Makefile | 3 +- scripts/run-unit-tests.sh | 28 ++++++++++ 5 files changed, 68 insertions(+), 128 deletions(-) create mode 100755 scripts/run-unit-tests.sh diff --git a/.buildkite/pipeline.quick.yml b/.buildkite/pipeline.quick.yml index de01bddf2..ff1dc177b 100644 --- a/.buildkite/pipeline.quick.yml +++ b/.buildkite/pipeline.quick.yml @@ -1,164 +1,108 @@ +env: + LANG: "en_GB.UTF-8" + steps: - label: macOS 11 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "macOS" commands: - - make bootstrap - - make analyze test + - ./scripts/run-unit-tests.sh PLATFORM=macOS artifact_paths: - - xcodebuild.log + - logs/* - label: macOS 10.14 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-10.14 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "macOS" commands: - - make bootstrap - - make analyze test + - ./scripts/run-unit-tests.sh PLATFORM=macOS artifact_paths: - - xcodebuild.log + - logs/* - label: iOS 13 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "iOS" - OS: "13.7" concurrency: 3 concurrency_group: cocoa-unit-tests commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=13.7 artifact_paths: - - xcodebuild.log + - logs/* - label: iOS 12 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "iOS" - OS: "12.4" concurrency: 3 concurrency_group: cocoa-unit-tests commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=12.4 artifact_paths: - - xcodebuild.log + - logs/* - label: iOS 11 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "iOS" - OS: "11.4" concurrency: 3 concurrency_group: cocoa-unit-tests commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=11.4 artifact_paths: - - xcodebuild.log + - logs/* - label: iOS 10 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-10.15 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "iOS" - OS: "10.3.1" - DEVICE: "iPhone 5s" commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=10.3.1 DEVICE=iPhone\ 5s artifact_paths: - - xcodebuild.log + - logs/* - label: tvOS 13 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "tvOS" - OS: "13.3" concurrency: 3 concurrency_group: cocoa-unit-tests commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=13.3 artifact_paths: - - xcodebuild.log + - logs/* - label: tvOS 12 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "tvOS" - OS: "12.4" concurrency: 3 concurrency_group: cocoa-unit-tests commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=12.4 artifact_paths: - - xcodebuild.log + - logs/* - label: tvOS 11 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "tvOS" - OS: "11.4" concurrency: 3 concurrency_group: cocoa-unit-tests commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=11.4 artifact_paths: - - xcodebuild.log + - logs/* - label: tvOS 10 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-10.15 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "tvOS" - OS: "10.2" commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=10.2 artifact_paths: - - xcodebuild.log + - logs/* - label: ':apple: macOS 10.15 full end-to-end tests' depends_on: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 539c5d0e4..a48146eb4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -30,8 +30,6 @@ steps: queue: opensource-mac-cocoa-10.15 concurrency: 3 concurrency_group: cocoa-unit-tests - env: - LANG: "en_GB.UTF-8" commands: - mkdir -p features/fixtures/carthage-proj - make build_swift @@ -56,88 +54,55 @@ steps: queue: opensource-mac-cocoa-10.15 concurrency: 3 concurrency_group: cocoa-unit-tests - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "macOS" commands: - - make bootstrap - - make analyze test + - ./scripts/run-unit-tests.sh PLATFORM=macOS + artifact_paths: + - logs/* - label: macOS 10.13 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-10.13 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "macOS" commands: - - make bootstrap - - make analyze test + - ./scripts/run-unit-tests.sh PLATFORM=macOS artifact_paths: - - xcodebuild.log + - logs/* - label: iOS 14 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "iOS" - OS: "14.3" commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=14.3 artifact_paths: - - xcodebuild.log + - logs/* - label: iOS 9 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-10.13 - env: - LANG: "en_GB.UTF-8" - PLATFORM: "iOS" - OS: "9.3" - DEVICE: "iPhone 5s" commands: - - make bootstrap - - TEST_CONFIGURATION=Debug make test - - TEST_CONFIGURATION=Release make test + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=9.3 DEVICE=iPhone\ 5s artifact_paths: - - xcodebuild.log + - logs/* - label: tvOS 14 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-11 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "tvOS" - OS: "14.3" commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=14.3 artifact_paths: - - xcodebuild.log + - logs/* - label: tvOS 9 unit tests timeout_in_minutes: 10 agents: queue: opensource-mac-cocoa-10.13 - env: - LANG: "en_GB.UTF-8" - TEST_CONFIGURATION: "Debug" - PLATFORM: "tvOS" - OS: "9.2" commands: - - make bootstrap - - make test + - ./scripts/run-unit-tests.sh PLATFORM=tvOS OS=9.2 artifact_paths: - - xcodebuild.log + - logs/* - label: ':ios: iOS 14 barebones end-to-end tests' depends_on: diff --git a/.gitignore b/.gitignore index 771fae6e6..27fee0f0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store build/ +logs/ *.pbxuser !default.pbxuser *.mode1v3 @@ -10,6 +11,7 @@ build/ !default.perspectivev3 xcuserdata *.xccheckout +*.xcresult *.moved-aside DerivedData *.hmap diff --git a/Makefile b/Makefile index 9eba26d6b..f6c07786e 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,9 @@ PLATFORM?=iOS OS?=latest TEST_CONFIGURATION?=Debug +XCODEBUILD_EXTRA_ARGS?= DATA_PATH=DerivedData -BUILD_FLAGS=-project Bugsnag.xcodeproj -scheme Bugsnag-$(PLATFORM) -derivedDataPath $(DATA_PATH) +BUILD_FLAGS=-project Bugsnag.xcodeproj -scheme Bugsnag-$(PLATFORM) -derivedDataPath $(DATA_PATH) $(XCODEBUILD_EXTRA_ARGS) ifeq ($(PLATFORM),macOS) SDK?=macosx diff --git a/scripts/run-unit-tests.sh b/scripts/run-unit-tests.sh new file mode 100755 index 000000000..967a5e5d8 --- /dev/null +++ b/scripts/run-unit-tests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -o nounset + +xcresult=$(date '+BugsnagTests-%Y-%m-%d-%H-%M-%S.xcresult') + +die() { + status=$? + echo "^^^ +++" + mkdir -p logs + [[ -f xcodebuild.log ]] && mv xcodebuild.log logs/ + [[ -d $xcresult ]] && zip -qr "logs/$xcresult.zip" "$xcresult" + exit $status +} + + +echo "--- Analyze" + +make analyze "$@" || die + +rm -rf DerivedData + + +echo "--- Test" + +make test XCODEBUILD_EXTRA_ARGS="-resultBundlePath $xcresult" "$@" || die + +rm -rf "$xcresult"