From 36cef60672fb7cf99b11d3d5fb584d89d5317643 Mon Sep 17 00:00:00 2001 From: wxbty Date: Tue, 9 Jan 2024 20:43:08 +0800 Subject: [PATCH] fix --- .github/workflows/build-and-bh-pr.yml | 432 ++++++++++++++++++++++++++ 1 file changed, 432 insertions(+) create mode 100644 .github/workflows/build-and-bh-pr.yml diff --git a/.github/workflows/build-and-bh-pr.yml b/.github/workflows/build-and-bh-pr.yml new file mode 100644 index 00000000000..2faa642416f --- /dev/null +++ b/.github/workflows/build-and-bh-pr.yml @@ -0,0 +1,432 @@ +name: Build and Test For PR + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +env: + FORK_COUNT: 2 + FAIL_FAST: 0 + SHOW_ERROR_DETAIL: 1 + #multi-version size limit + VERSIONS_LIMIT: 4 + JACOCO_ENABLE: true + CANDIDATE_VERSIONS: ' + spring.version:5.3.24; + spring-boot.version:2.7.6; + ' + +jobs: + license: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check License + uses: apache/skywalking-eyes@e1a02359b239bd28de3f6d35fdc870250fa513d5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Set up JDK 21" + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 21 + - name: "Compile Dubbo (Linux)" + run: | + ./mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast -T 2C clean install -DskipTests=true -DskipIntegrationTests=true -Dcheckstyle.skip=true -Dcheckstyle_unix.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true + - name: Check Dependencies' License + uses: apache/skywalking-eyes/dependency@e1a02359b239bd28de3f6d35fdc870250fa513d5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + config: .licenserc.yaml + mode: check + + build-source: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.dubbo-version.outputs.version }} + steps: + - uses: actions/checkout@v3 + with: + path: dubbo + - uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 17 + - uses: actions/cache@v3 + name: "Cache local Maven repository" + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-maven- + - name: "Dubbo cache" + uses: actions/cache@v3 + with: + path: ~/.m2/repository/org/apache/dubbo + key: ${{ runner.os }}-dubbo-snapshot-${{ github.sha }}-${{ github.run_id }} + - name: "Build Dubbo with Maven" + run: | + cd ./dubbo + ./mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast clean source:jar install -Pjacoco,checkstyle -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.test.skip=true -Dmaven.test.skip.exec=true -DembeddedZookeeperPath=${{ github.workspace }}/.tmp/zookeeper + - name: "Pack class result" + run: | + shopt -s globstar + zip ${{ github.workspace }}/class.zip **/target/classes/* -r + - name: "Upload class result" + uses: actions/upload-artifact@v3 + with: + name: "class-file" + path: ${{ github.workspace }}/class.zip + - name: "Pack checkstyle file if failure" + if: failure() + run: zip ${{ github.workspace }}/checkstyle.zip *checkstyle* -r + - name: "Upload checkstyle file if failure" + if: failure() + uses: actions/upload-artifact@v3 + with: + name: "checkstyle-file" + path: ${{ github.workspace }}/checkstyle.zip + - name: "Calculate Dubbo Version" + id: dubbo-version + run: | + REVISION=`awk '/[^<]+<\/revision>/{gsub(/|<\/revision>/,"",$1);print $1;exit;}' ./dubbo/pom.xml` + echo "version=$REVISION" >> $GITHUB_OUTPUT + echo "dubbo version: $REVISION" + + + unit-test: + needs: [build-source, unit-test-prepare] + name: "Unit Test On ubuntu-latest" + runs-on: ubuntu-latest + strategy: + fail-fast: false + env: + DISABLE_FILE_SYSTEM_TEST: true + CURRENT_ROLE: ${{ matrix.case-role }} + DUBBO_DEFAULT_SERIALIZATION: fastjson2 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: "Set up JDK ${{ matrix.jdk }}" + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 21 + - uses: actions/cache@v3 + name: "Cache local Maven repository" + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-maven- + - uses: actions/cache@v3 + name: "Cache zookeeper binary archive" + id: "cache-zookeeper" + with: + path: ${{ github.workspace }}/.tmp/zookeeper + key: zookeeper-${{ runner.os }}-${{ env.ZOOKEEPER_VERSION }} + restore-keys: | + zookeeper-${{ runner.os }}- + - uses: actions/cache@v3 + name: "Cache secret key" + id: "cache-secret-cert" + with: + path: ${{ github.workspace }}/.tmp/rsa + key: secret-rsa-${{ runner.os }}-${{ github.run_id }} + - name: "Get sonarcloud token" + if: ${{ github.repository == 'apache/dubbo' }} + run: | + curl "http://dubbo-vm.apache.org:8000/token?workflow_id=${{ github.run_id }}" -o ${{ github.workspace }}/.tmp/encrypted-sonarcloud-token + openssl rsautl -decrypt -in ${{ github.workspace }}/.tmp/encrypted-sonarcloud-token -out ${{ github.workspace }}/.tmp/decrypted-sonarcloud-token -inkey ${{ github.workspace }}/.tmp/rsa/rsa_private.pem + - name: "Test with Maven with SonarCloud Scan" + if: ${{ github.repository == 'apache/dubbo' }} + timeout-minutes: 90 + env: + # Needed to get some information about the pull request, if any + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + source ${{ github.workspace }}/.tmp/decrypted-sonarcloud-token + ./mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast clean test verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pjacoco,jdk15ge-simple,'!jdk15ge',jacoco089 -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_dubbo -DtrimStackTrace=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Dmaven.wagon.http.retryHandler.count=5 -DskipTests=false -DskipIntegrationTests=false -Dcheckstyle.skip=false -Dcheckstyle_unix.skip=false -Drat.skip=false -Dmaven.javadoc.skip=true -DembeddedZookeeperPath=${{ github.workspace }}/.tmp/zookeeper -Dsonar.coverage.jacoco.xmlReportPaths=dubbo-test/dubbo-dependencies-all/target/site/jacoco-aggregate/jacoco.xml -Dsonar.login=${SONAR_TOKEN} + - name: "Test with Maven without SonarCloud Scan" + if: ${{ github.repository != 'apache/dubbo' }} + timeout-minutes: 90 + run: | + ./mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast clean test verify -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Pjacoco,jdk15ge-simple,'!jdk15ge',jacoco089 -DtrimStackTrace=false -Dmaven.wagon.http.retryHandler.count=5 -DskipTests=false -DskipIntegrationTests=false -Dcheckstyle.skip=false -Dcheckstyle_unix.skip=false -Drat.skip=false -Dmaven.javadoc.skip=true -DembeddedZookeeperPath=${{ github.workspace }}/.tmp/zookeeper + - name: "Upload coverage result" + uses: actions/upload-artifact@v3 + with: + name: coverage-result + path: "**/target/site/**/jacoco.xml" + + integration-test-prepare: + runs-on: ubuntu-latest + env: + JOB_COUNT: 3 + steps: + - uses: actions/checkout@v3 + with: + repository: 'apache/dubbo-samples' + ref: master + - name: "Prepare test list" + run: | + bash ./test/scripts/prepare-test.sh + - name: "Upload test list" + uses: actions/upload-artifact@v3 + with: + name: test-list + path: test/jobs + + integration-test-job: + needs: [build-source, integration-test-prepare] + name: "Integration Test on ubuntu-latest (JobId: ${{matrix.job_id}})" + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + JAVA_VER: 8 + TEST_CASE_FILE: jobs/testjob_${{matrix.job_id}}.txt + strategy: + fail-fast: false + matrix: + job_id: [1, 2, 3] + steps: + - uses: actions/checkout@v3 + with: + repository: 'apache/dubbo-samples' + ref: master + - name: "Cache local Maven repository" + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-maven- + - name: "Restore Dubbo cache" + uses: actions/cache@v3 + with: + path: ~/.m2/repository/org/apache/dubbo + key: ${{ runner.os }}-dubbo-snapshot-${{ github.sha }}-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-dubbo-snapshot-${{ github.sha }} + ${{ runner.os }}-dubbo-snapshot- + - name: "Download test list" + uses: actions/download-artifact@v3 + with: + name: test-list + path: test/jobs/ + - name: "Set up JDK 8" + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 8 + - name: "Init Candidate Versions" + run: | + DUBBO_VERSION="${{needs.build-source.outputs.version}}" + CANDIDATE_VERSIONS="dubbo.version:$DUBBO_VERSION;compiler.version:$DUBBO_VERSION;$CANDIDATE_VERSIONS;dubbo.compiler.version:$DUBBO_VERSION" + echo "CANDIDATE_VERSIONS=$CANDIDATE_VERSIONS" >> $GITHUB_ENV + - name: "Build test image" + run: | + cd test && bash ./build-test-image.sh + - name: "Run tests" + run: cd test && bash ./run-tests.sh + - name: "merge jacoco resule" + run: | + cd test/dubbo-test-jacoco-merger && mvn clean compile exec:java -Dexec.mainClass="org.apache.dubbo.test.JacocoMerge" -Dexec.args="${{github.workspace}}" + - name: "Upload jacoco" + uses: actions/upload-artifact@v3 + with: + name: jacoco-result + path: target/jacoco*.exec + - name: "Upload test result" + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-result + path: test/jobs/*-result* + + integration-test-result: + needs: [integration-test-job] + if: always() + runs-on: ubuntu-latest + env: + JAVA_VER: 8 + steps: + - uses: actions/checkout@v3 + with: + repository: 'apache/dubbo-samples' + ref: master + - name: "Download test result" + uses: actions/download-artifact@v3 + with: + name: test-result + path: test/jobs/ + - name: "Merge test result" + run: ./test/scripts/merge-test-results.sh + + jacoco-result-merge: + runs-on: ubuntu-latest + needs: [integration-test-result, unit-test] + steps: + - uses: actions/checkout@v3 + with: + path: "./dubbo" + - uses: actions/checkout@v3 + with: + repository: 'apache/dubbo-samples' + path: "./dubbo-samples" + - name: "Set up JDK 21" + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 21 + - name: "Restore class result" + uses: actions/download-artifact@v3 + with: + name: "class-file" + path: ${{ github.workspace }} + - name: "Unpack class result" + run: | + cd ${{ github.workspace }}/dubbo + unzip -o ${{ github.workspace }}/class.zip + - name: "Restore jacoco exec" + uses: actions/download-artifact@v3 + with: + name: jacoco-result + path: dubbo-samples/target/ + - name: "Merge jacoco result" + run: | + cd ${{ github.workspace }}/dubbo-samples/test/dubbo-test-jacoco-merger + mvn clean compile exec:java -Dexec.mainClass="org.apache.dubbo.test.JacocoReport" -Dexec.args="${{github.workspace}}/dubbo-samples ${{github.workspace}}/dubbo" + - name: "Restore coverage result" + uses: actions/download-artifact@v3 + with: + name: coverage-result + path: dubbo/ + - name: "Upload coverage to Codecov" + uses: codecov/codecov-action@v3 + with: + verbose: true + + error-code-inspecting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + path: "./dubbo" + + - uses: actions/checkout@v3 + with: + repository: 'apache/dubbo-test-tools' + ref: main + path: "./dubbo-test-tools" + + - name: "Set up JDK 21" + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 21 + + - name: "Compile Dubbo (Linux)" + run: | + cd ${{ github.workspace }}/dubbo + ./mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast -T 2C clean install -DskipTests=true -DskipIntegrationTests=true -Dcheckstyle.skip=true -Dcheckstyle_unix.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true + - name: "Run Error Code Inspecting" + env: + DUBBO_ECI_REPORT_AS_ERROR: true + run: | + cd ${{ github.workspace }}/dubbo-test-tools/dubbo-error-code-inspector + ../mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast -T 2C package exec:java -Ddubbo.eci.report-as-error=${DUBBO_ECI_REPORT_AS_ERROR} -Dmaven.test.skip=true -Dmaven.test.skip.exec=true -Ddubbo.eci.path=${{ github.workspace }}/dubbo + + - name: "Upload error code inspection result" + # always() should not be used here, since we don't need to handle the 'canceled' situation. + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: "error-inspection-result" + path: ${{ github.workspace }}/dubbo-test-tools/dubbo-error-code-inspector/error-inspection-result.txt + + native-image-inspecting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + path: "./dubbo" + + - name: "Setup GraalVM environment" + uses: graalvm/setup-graalvm@v1 + with: + version: '22.3.0' + java-version: '17' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} + native-image-job-reports: 'true' + + - name: "Setup Zookeeper environment" + run: | + wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.3/apache-zookeeper-3.8.3-bin.tar.gz + tar -zxvf apache-zookeeper-3.8.3-bin.tar.gz + mv apache-zookeeper-3.8.3-bin/conf/zoo_sample.cfg apache-zookeeper-3.8.3-bin/conf/zoo.cfg + apache-zookeeper-3.8.3-bin/bin/zkServer.sh start + + - name: "Check environment" + run: | + java --version + native-image --version + + - name: "Compile Dubbo (Linux)" + run: | + cd ${{ github.workspace }}/dubbo + ./mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast -T 2C clean install -DskipTests=true -DskipIntegrationTests=true -Dcheckstyle.skip=true -Dcheckstyle_unix.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true + + - name: "Compile and run Dubbo demo for native (Linux)" + run: | + cd ${{ github.workspace }}/dubbo/dubbo-demo/dubbo-demo-native/dubbo-demo-native-provider + ${{ github.workspace }}/dubbo/mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast -T 2C clean package -P native -Dmaven.test.skip=true native:compile + nohup ./target/dubbo-demo-native-provider & + cd ${{ github.workspace }}/dubbo/dubbo-demo/dubbo-demo-native/dubbo-demo-native-consumer + ${{ github.workspace }}/dubbo/mvnw --batch-mode --no-snapshot-updates -e --no-transfer-progress --fail-fast -T 2C clean package -P native -Dmaven.test.skip=true native:compile + ./target/dubbo-demo-native-consumer + + integration-benchmark-prepare: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + repository: 'wxbty/dubbo-benchmark' + ref: continuous + + integration-benchmark-job: + needs: [build-source, integration-benchmark-prepare] + name: "Integration Benchmark on ubuntu-latest (JobId: ${{matrix.job_id}})" + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + JAVA_VER: 8 + TEST_CASE_FILE: jobs/bh_job_${{matrix.job_id}}.txt + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + with: + repository: 'wxbty/dubbo-benchmark' + ref: continuous + - name: "Set up JDK 8" + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 8 + - name: "Init Candidate Versions" + run: | + DUBBO_VERSION="${{needs.build-source.outputs.version}}" + CANDIDATE_VERSIONS="dubbo.version:$DUBBO_VERSION;compiler.version:$DUBBO_VERSION;$CANDIDATE_VERSIONS;dubbo.compiler.version:$DUBBO_VERSION" + echo "CANDIDATE_VERSIONS=$CANDIDATE_VERSIONS" >> $GITHUB_ENV + - name: "Run bh server" + run: | + ./benchmark.sh dubbo-metrics-server/ + - name: "Run bh client" + run: cd test && bash ./run-dubbo-benchmark.sh