-
-
Notifications
You must be signed in to change notification settings - Fork 318
144 lines (138 loc) · 6.09 KB
/
benchmarking.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
name: Benchmarking
on:
schedule:
- cron: '0 0 * * *' # every night at midnight UTC
push:
branches:
- main
pull_request:
paths:
# test changes to Sentry SDK sources
- 'Sources/**'
# test changes to benchmarking implementation
- 'Samples/iOS-Swift/iOS-Swift/**'
- 'Samples/iOS-Swift/PerformanceBenchmarks/**'
- '.github/workflows/benchmarking.yml'
- '.sauce/benchmarking-config.yml'
- 'fastlane/**'
- 'scripts/ci-select-xcode.sh'
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
build-benchmark-test-target:
name: Build app and test runner
runs-on: macos-13-xlarge
steps:
- uses: actions/checkout@v4
- run: ./scripts/ci-select-xcode.sh
- uses: ruby/setup-ruby@v1
with:
bundler-cache: true
- name: Install SentryCli
run: brew install getsentry/tools/sentry-cli
- name: Cache iOS-Swift App and dSYM build products
id: ios-swift-cache
uses: actions/cache@v4
with:
path: |
DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/iOS-Swift/**') }}-${{ hashFiles('Sources/Sentry/**') }}
- name: Cache iOS-Swift UI Test Runner App build product
id: ios-swift-benchmark-runner-cache
uses: actions/cache@v4
with:
path: |
DerivedData/Build/Products/Debug-iphoneos/PerformanceBenchmarks-Runner.app
key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/PerformanceBenchmarks/**') }}
- run: bundle exec fastlane build_ios_swift_for_tests
env:
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
- run: bundle exec fastlane build_ios_benchmark_test
env:
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
- name: Upload dSYMs
run: |
sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
- name: Archiving DerivedData
uses: actions/upload-artifact@v4
with:
name: DerivedData-Xcode
path: |
**/Debug-iphoneos/iOS-Swift.app
**/Debug-iphoneos/PerformanceBenchmarks-Runner.app
run-ui-tests-with-sauce:
name: Run benchmarks on Sauce Labs
runs-on: ubuntu-latest
needs: build-benchmark-test-target
strategy:
fail-fast: false
matrix:
suite: ['High-end device', 'Mid-range device', 'Low-end device']
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: DerivedData-Xcode
- run: npm install -g [email protected]
- name: Run Benchmarks in SauceLab
env:
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
run: saucectl run --select-suite "${{matrix.suite}}" --config .sauce/benchmarking-config.yml --tags benchmark
app-metrics:
name: Collect app metrics
runs-on: macos-13-xlarge
steps:
- name: Git checkout
uses: actions/checkout@v4
- run: ./scripts/ci-select-xcode.sh
- uses: ruby/setup-ruby@v1
with:
bundler-cache: true
- uses: actions/cache@v4
id: app-plain-cache
with:
path: Tests/Perf/test-app-plain.ipa
key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('fastlane/Fastfile', 'Tests/Perf/test-app-plain/**') }}
- name: Build test app plain
if: steps.app-plain-cache.outputs['cache-hit'] != 'true'
run: bundle exec fastlane build_perf_test_app_plain
env:
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
- name: Build test app with sentry
run: bundle exec fastlane build_perf_test_app_sentry
env:
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
- name: Collect app metrics
uses: getsentry/action-app-sdk-overhead-metrics@v1
with:
config: Tests/Perf/metrics-test.yml
sauce-user: ${{ secrets.SAUCE_USERNAME }}
sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}