CI: add random tests for meta #831
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: mutate-test | |
| on: | |
| pull_request: | |
| branches: | |
| - 'main' | |
| paths: | |
| - '**/*_test.go' | |
| workflow_dispatch: | |
| inputs: | |
| test_file: | |
| type: string | |
| description: "the go test file relative path you want to mutate, eg cmd/meta/xattr_test.go" | |
| required: true | |
| default: "" | |
| job_total: | |
| type: string | |
| description: "number of job to run mutation test" | |
| required: true | |
| default: "1" | |
| debug: | |
| type: boolean | |
| description: "Run the build with tmate debugging enabled" | |
| required: false | |
| default: false | |
| jobs: | |
| build-matrix: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 'oldstable' | |
| - name: install go-mutesting | |
| run: | | |
| go install github.com/zimmski/go-mutesting/cmd/go-mutesting@latest | |
| - id: set-matrix | |
| run: | | |
| sudo .github/scripts/apt_install.sh jq | |
| if [ "${{github.event_name}}" == "pull_request" ]; then | |
| echo github.event.pull_request.base.sha is ${{github.event.pull_request.base.sha}} | |
| echo github.event.pull_request.head.sha is ${{github.event.pull_request.head.sha}} | |
| echo github.sha is ${{ github.sha }} | |
| changed_file_str=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep _test.go$ | xargs) | |
| echo "added or changed test files: $changed_file_str" | |
| changed_file_array=($changed_file_str) | |
| declare -a Jobs=(); | |
| for test_file_name in "${changed_file_array[@]}" | |
| do | |
| echo "test_file_name is $test_file_name" | |
| if grep -q "//mutate:disable" $test_file_name; then | |
| echo "found //mutate:disable in $test_file_name" | |
| continue | |
| fi | |
| source_file_name=${test_file_name%"_test.go"}.go | |
| echo "source_file_name is :" $source_file_name | |
| black_list_file=black.list | |
| TEST_FILE_NAME="$test_file_name" BLACK_LIST_FILE=$black_list_file python3 .github/scripts/mutate/parse_black_list.py | |
| echo "black list checksum: " | |
| cat $black_list_file | |
| total_count=$(go-mutesting $source_file_name --debug --no-exec --blacklist $black_list_file| grep "Save mutation into" | wc -l) | |
| echo "total_count is $total_count" | |
| job_total=$(TEST_FILE_NAME=$test_file_name python3 .github/scripts/mutate/parse_job_total.py) | |
| echo "job_total specified: $job_total" | |
| if [ $job_total -eq 0 ]; then | |
| if [ $total_count -gt 200 ]; then | |
| job_total=4 | |
| else | |
| job_total=1 | |
| fi | |
| fi | |
| echo "job_total: $job_total" | |
| for i in `seq 1 $job_total` | |
| do | |
| Jobs=("${Jobs[@]}" "$test_file_name-$i-$job_total") | |
| done | |
| done | |
| value=`printf '%s\n' "${Jobs[@]}" | jq -R . | jq -cs .` | |
| echo "value: $value" | |
| echo "matrix=$value" >> $GITHUB_OUTPUT | |
| elif [ "${{github.event_name}}" == "workflow_dispatch" ]; then | |
| test_file_name=${{github.event.inputs.test_file}} | |
| echo "test file is $test_file_name" | |
| job_total=${{github.event.inputs.job_total}} | |
| echo "job_total is $job_total" | |
| declare -a Jobs=(); | |
| for i in `seq 1 $job_total` | |
| do | |
| Jobs=("${Jobs[@]}" "$test_file_name-$i-$job_total") | |
| done | |
| value=`printf '%s\n' "${Jobs[@]}" | jq -R . | jq -cs .` | |
| echo "value: $value" | |
| echo "matrix=$value" >> $GITHUB_OUTPUT | |
| fi | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| mutate-test: | |
| if: "!github.event.pull_request.draft" | |
| name: ${{matrix.test_file}} | |
| needs: build-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_file: ${{ fromJson(needs.build-matrix.outputs.matrix) }} | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Get Current Job Log URL | |
| uses: Tiryoh/gha-jobid-action@v0 | |
| id: jobs | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| job_name: ${{matrix.test_file}} | |
| - name: Build | |
| uses: ./.github/actions/build | |
| - name: Remove problem matcher for go | |
| run: | | |
| # https://github.com/actions/setup-go/blob/main/matchers.json | |
| echo "::remove-matcher owner=go::" | |
| - name: Install Packages | |
| run: | | |
| go install github.com/zimmski/go-mutesting/cmd/go-mutesting@latest | |
| sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr python3-tk | |
| sudo pip install mysqlclient | |
| - name: Prepare Database | |
| timeout-minutes: 10 | |
| run: | | |
| docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" | |
| go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket | |
| make | |
| sudo make -C fstests setup | |
| - name: run mutate test | |
| # timeout-minutes: 120 | |
| run: | | |
| sudo chmod 777 /var/jfsCache | |
| test_file=$(echo ${{matrix.test_file}} | awk -F'-' '{print $1}') | |
| job_index=$(echo ${{matrix.test_file}} | awk -F'-' '{print $2}') | |
| job_total=$(echo ${{matrix.test_file}} | awk -F'-' '{print $3}') | |
| echo "test file is: $test_file, job_index is $job_index, job_total is $job_total" | |
| if [ -z "$test_file" ]; then | |
| echo "test file is empty, will not run mutate test" | |
| exit 0 | |
| fi | |
| source_file=${test_file%"_test.go"}.go | |
| echo "source file is :" $source_file | |
| package_path=$(dirname $test_file) | |
| echo "package path is :" $package_path | |
| test_cases=$(TEST_FILE_NAME=$test_file python3 .github/scripts/mutate/parse_test_cases.py || true) | |
| if [ "$?" -ne 0 ]; then | |
| echo "no test cases in test file, will not run mutate test" | |
| exit 0 | |
| fi | |
| echo "test cases: $test_cases" | |
| if [[ "$test_file" =~ ^pkg/.* ]]; then | |
| go test ./$package_path/... -v -run "$test_cases" -count=1 -cover -timeout=5m -coverpkg=./$package_path/... -coverprofile=mutest-cov.out | |
| elif [[ "$test_file" =~ ^cmd/.* ]]; then | |
| sudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword go test ./cmd/... -v -run "$test_cases" -count=1 -cover -timeout=5m -coverpkg=./pkg/...,./cmd/... -coverprofile=mutest-cov.out | |
| else | |
| echo "test file location error: $test_file" | |
| exit 0 | |
| fi | |
| black_list_file=black.list | |
| TEST_FILE_NAME="$test_file" BLACK_LIST_FILE=$black_list_file python3 .github/scripts/mutate/parse_black_list.py | |
| echo "black list checksum: " | |
| cat $black_list_file | |
| go-mutesting $source_file --debug --no-exec --do-not-remove-tmp-folder --blacklist $black_list_file | tee -a mutate.log | |
| mutation_dir=$(cat mutate.log | grep "Save mutations into" | awk -F' ' '{print $4}' | sed -e 's:"::g') | |
| echo "mutation dir is $mutation_dir" | |
| JOB_INDEX=$job_index JOB_TOTAL=$job_total MUTATE_ORIGINAL=$source_file MUTATION_DIR=$mutation_dir COVERAGE_FILE=mutest-cov.out TEST_FILE_NAME="$test_file" PACKAGE_PATH="$package_path" STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/mutesting.py | |
| # COVERAGE_FILE=mutest-cov.out TEST_FILE_NAME="$test_file" PACKAGE_PATH="$package_path" go-mutesting $source_file --debug --exec=.github/scripts/mutate/mutest.sh --do-not-remove-tmp-folder --blacklist $black_list_file | |
| if [ $? != 0 ]; then echo "run mutesting.py failed" && exit 1; fi | |
| [[ -z "${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} " ]] && echo "<WARNING>: MYSQL_PASSWORD is empty" && exit 0 | |
| export MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} | |
| JOB_NAME=${{matrix.test_file}} JOB_URL=${{steps.jobs.outputs.html_url}} STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/save_report.py | |
| - name: Setup upterm session | |
| if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) | |
| timeout-minutes: 60 | |
| uses: lhotari/action-upterm@v1 | |
| success-all-test: | |
| runs-on: ubuntu-20.04 | |
| needs: [build-matrix, mutate-test] | |
| if: always() && !github.event.pull_request.draft | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@v3 | |
| - uses: actions/checkout@v3 | |
| - name: Check Failure | |
| if: env.WORKFLOW_CONCLUSION == 'failure' | |
| run: exit 1 | |
| - name: Install tools | |
| run: | | |
| sudo pip install mysqlclient | |
| - name: Generate mutate report | |
| run: | | |
| [[ -z "${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} " ]] && echo "<WARNING>: MYSQL_PASSWORD is empty" && exit 0 | |
| export MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} | |
| mutate_report=$(python3 .github/scripts/mutate/query_report.py) | |
| echo "mutate_report is $mutate_report" | |
| # echo "mutate_report=$mutate_report" >> $GITHUB_ENV | |
| MY_STRING=$(cat << EOF | |
| $mutate_report | |
| EOF | |
| ) | |
| echo "MY_STRING<<EOF" >> $GITHUB_ENV | |
| echo "$MY_STRING" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - uses: mshick/add-pr-comment@v2 | |
| with: | |
| allow-repeats: true | |
| message: | | |
| *Mutate Test Report* | |
| ${{env.MY_STRING}} | |
| Usage: https://github.com/juicedata/juicefs/blob/main/.github/scripts/mutate/how_to_use_mutate_test.md | |
| - name: Send Slack Notification | |
| if: failure() && github.event_name != 'workflow_dispatch' | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |
| - name: Success | |
| if: success() | |
| run: echo "All Done" | |