-
Notifications
You must be signed in to change notification settings - Fork 929
256 lines (227 loc) · 10.4 KB
/
mutate-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
name: mutate-test
on:
pull_request:
branches:
- 'main'
paths:
- '**/*_test.go'
workflow_dispatch:
inputs:
test_file:
type: string
description: "the go test file relative path you want to mutate, eg cmd/meta/xattr_test.go"
required: true
default: ""
job_total:
type: string
description: "number of job to run mutation test"
required: true
default: "1"
debug:
type: boolean
description: "Run the build with tmate debugging enabled"
required: false
default: false
jobs:
build-matrix:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 'oldstable'
- name: install go-mutesting
run: |
go install github.com/zimmski/go-mutesting/cmd/go-mutesting@latest
- id: set-matrix
run: |
sudo .github/scripts/apt_install.sh jq
if [ "${{github.event_name}}" == "pull_request" ]; then
echo github.event.pull_request.base.sha is ${{github.event.pull_request.base.sha}}
echo github.event.pull_request.head.sha is ${{github.event.pull_request.head.sha}}
echo github.sha is ${{ github.sha }}
changed_file_str=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep _test.go$ | xargs)
echo "added or changed test files: $changed_file_str"
changed_file_array=($changed_file_str)
declare -a Jobs=();
for test_file_name in "${changed_file_array[@]}"
do
echo "test_file_name is $test_file_name"
if grep -q "//mutate:disable" $test_file_name; then
echo "found //mutate:disable in $test_file_name"
continue
fi
source_file_name=${test_file_name%"_test.go"}.go
echo "source_file_name is :" $source_file_name
black_list_file=black.list
TEST_FILE_NAME="$test_file_name" BLACK_LIST_FILE=$black_list_file python3 .github/scripts/mutate/parse_black_list.py
echo "black list checksum: "
cat $black_list_file
total_count=$(go-mutesting $source_file_name --debug --no-exec --blacklist $black_list_file| grep "Save mutation into" | wc -l)
echo "total_count is $total_count"
job_total=$(TEST_FILE_NAME=$test_file_name python3 .github/scripts/mutate/parse_job_total.py)
echo "job_total specified: $job_total"
if [ $job_total -eq 0 ]; then
if [ $total_count -gt 200 ]; then
job_total=4
else
job_total=1
fi
fi
echo "job_total: $job_total"
for i in `seq 1 $job_total`
do
Jobs=("${Jobs[@]}" "$test_file_name-$i-$job_total")
done
done
value=`printf '%s\n' "${Jobs[@]}" | jq -R . | jq -cs .`
echo "value: $value"
echo "matrix=$value" >> $GITHUB_OUTPUT
elif [ "${{github.event_name}}" == "workflow_dispatch" ]; then
test_file_name=${{github.event.inputs.test_file}}
echo "test file is $test_file_name"
job_total=${{github.event.inputs.job_total}}
echo "job_total is $job_total"
declare -a Jobs=();
for i in `seq 1 $job_total`
do
Jobs=("${Jobs[@]}" "$test_file_name-$i-$job_total")
done
value=`printf '%s\n' "${Jobs[@]}" | jq -R . | jq -cs .`
echo "value: $value"
echo "matrix=$value" >> $GITHUB_OUTPUT
fi
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
mutate-test:
timeout-minutes: 120
if: "!github.event.pull_request.draft"
name: ${{matrix.test_file}}
needs: build-matrix
strategy:
fail-fast: false
matrix:
test_file: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
runs-on: ubuntu-20.04
permissions:
pull-requests: write
steps:
- uses: actions/checkout@v3
- name: Get Current Job Log URL
uses: Tiryoh/gha-jobid-action@v0
id: jobs
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
job_name: ${{matrix.test_file}}
- name: Build
timeout-minutes: 10
uses: ./.github/actions/build
- name: Remove problem matcher for go
run: |
# https://github.com/actions/setup-go/blob/main/matchers.json
echo "::remove-matcher owner=go::"
- name: Install Packages
run: |
go install github.com/zimmski/go-mutesting/cmd/go-mutesting@latest
sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr python3-tk
sudo pip install mysqlclient
- name: Prepare Database
timeout-minutes: 10
run: |
docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001"
go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket
make
sudo make -C fstests setup
- name: run mutate test
# timeout-minutes: 120
run: |
sudo chmod 777 /var/jfsCache
test_file=$(echo ${{matrix.test_file}} | awk -F'-' '{print $1}')
job_index=$(echo ${{matrix.test_file}} | awk -F'-' '{print $2}')
job_total=$(echo ${{matrix.test_file}} | awk -F'-' '{print $3}')
echo "test file is: $test_file, job_index is $job_index, job_total is $job_total"
if [ -z "$test_file" ]; then
echo "test file is empty, will not run mutate test"
exit 0
fi
source_file=${test_file%"_test.go"}.go
echo "source file is :" $source_file
package_path=$(dirname $test_file)
echo "package path is :" $package_path
test_cases=$(TEST_FILE_NAME=$test_file python3 .github/scripts/mutate/parse_test_cases.py || true)
if [ "$?" -ne 0 ]; then
echo "no test cases in test file, will not run mutate test"
exit 0
fi
echo "test cases: $test_cases"
if [[ "$test_file" =~ ^pkg/.* ]]; then
go test ./$package_path/... -v -run "$test_cases" -count=1 -cover -timeout=5m -coverpkg=./$package_path/... -coverprofile=mutest-cov.out
elif [[ "$test_file" =~ ^cmd/.* ]]; then
sudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword go test ./cmd/... -v -run "$test_cases" -count=1 -cover -timeout=5m -coverpkg=./pkg/...,./cmd/... -coverprofile=mutest-cov.out
else
echo "test file location error: $test_file"
exit 0
fi
black_list_file=black.list
TEST_FILE_NAME="$test_file" BLACK_LIST_FILE=$black_list_file python3 .github/scripts/mutate/parse_black_list.py
echo "black list checksum: "
cat $black_list_file
go-mutesting $source_file --debug --no-exec --do-not-remove-tmp-folder --blacklist $black_list_file | tee -a mutate.log
mutation_dir=$(cat mutate.log | grep "Save mutations into" | awk -F' ' '{print $4}' | sed -e 's:"::g')
echo "mutation dir is $mutation_dir"
JOB_INDEX=$job_index JOB_TOTAL=$job_total MUTATE_ORIGINAL=$source_file MUTATION_DIR=$mutation_dir COVERAGE_FILE=mutest-cov.out TEST_FILE_NAME="$test_file" PACKAGE_PATH="$package_path" STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/mutesting.py
# COVERAGE_FILE=mutest-cov.out TEST_FILE_NAME="$test_file" PACKAGE_PATH="$package_path" go-mutesting $source_file --debug --exec=.github/scripts/mutate/mutest.sh --do-not-remove-tmp-folder --blacklist $black_list_file
if [ $? != 0 ]; then echo "run mutesting.py failed" && exit 1; fi
[[ -z "${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} " ]] && echo "<WARNING>: MYSQL_PASSWORD is empty" && exit 0
export MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
JOB_NAME=${{matrix.test_file}} JOB_URL=${{steps.jobs.outputs.html_url}} STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/save_report.py
- name: Setup upterm session
if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
timeout-minutes: 60
uses: lhotari/action-upterm@v1
success-all-test:
runs-on: ubuntu-20.04
needs: [build-matrix, mutate-test]
if: always() && !github.event.pull_request.draft
steps:
- uses: technote-space/workflow-conclusion-action@v3
- uses: actions/checkout@v3
- name: Check Failure
if: env.WORKFLOW_CONCLUSION == 'failure'
run: exit 1
- name: Install tools
run: |
sudo pip install mysqlclient
- name: Generate mutate report
run: |
[[ -z "${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} " ]] && echo "<WARNING>: MYSQL_PASSWORD is empty" && exit 0
export MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
mutate_report=$(python3 .github/scripts/mutate/query_report.py)
echo "mutate_report is $mutate_report"
# echo "mutate_report=$mutate_report" >> $GITHUB_ENV
MY_STRING=$(cat << EOF
$mutate_report
EOF
)
echo "MY_STRING<<EOF" >> $GITHUB_ENV
echo "$MY_STRING" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- uses: mshick/add-pr-comment@v2
with:
allow-repeats: true
message: |
*Mutate Test Report*
${{env.MY_STRING}}
Usage: https://github.com/juicedata/juicefs/blob/main/.github/scripts/mutate/how_to_use_mutate_test.md
- name: Send Slack Notification
if: failure() && github.event_name != 'workflow_dispatch'
uses: juicedata/slack-notify-action@main
with:
channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
- name: Success
if: success()
run: echo "All Done"