feat(query): Reduce serialisation memory usage when spilling to local disk #1619
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cloud | |
| on: | |
| pull_request_target: | |
| types: | |
| - labeled | |
| branches: | |
| - main | |
| permissions: | |
| id-token: write | |
| pull-requests: write | |
| contents: read | |
| env: | |
| BUILD_PROFILE: release | |
| RUNNER_PROVIDER: aws | |
| jobs: | |
| info: | |
| if: contains(github.event.pull_request.labels.*.name, 'ci-cloud') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark-cloud') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark-load') || contains(github.event.pull_request.labels.*.name, 'ci-benchmark-local') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| sha: ${{ steps.sha.outputs.sha }} | |
| target: ${{ steps.sha.outputs.target }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: "refs/pull/${{ github.event.number }}/merge" | |
| - name: Get SHA | |
| shell: bash | |
| id: sha | |
| run: | | |
| echo "sha=$(git rev-parse --verify HEAD)" > $GITHUB_OUTPUT | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark') }}; then | |
| echo "target=all" >> $GITHUB_OUTPUT | |
| elif ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark-cloud') }}; then | |
| echo "target=cloud" >> $GITHUB_OUTPUT | |
| elif ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark-load') }}; then | |
| echo "target=load" >> $GITHUB_OUTPUT | |
| elif ${{ contains(github.event.pull_request.labels.*.name, 'ci-benchmark-local') }}; then | |
| echo "target=local" >> $GITHUB_OUTPUT | |
| fi | |
| build: | |
| needs: info | |
| runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] | |
| strategy: | |
| matrix: | |
| include: | |
| - { arch: x86_64, runner: X64 } | |
| - { arch: aarch64, runner: ARM64 } | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: "refs/pull/${{ github.event.number }}/merge" | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| timeout-minutes: 60 | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| target: ${{ matrix.arch }}-unknown-linux-gnu | |
| artifacts: meta,query | |
| category: udf | |
| features: python-udf | |
| docker: | |
| needs: [info, build] | |
| timeout-minutes: 10 | |
| runs-on: [self-hosted, X64, Linux, 4c8g, aws] | |
| outputs: | |
| tag: ${{ steps.prepare.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download artifact for x86_64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| target: x86_64-unknown-linux-gnu | |
| path: ./distro/linux/amd64/ | |
| category: udf | |
| artifacts: query | |
| - name: Download artifact for aarch64 | |
| uses: ./.github/actions/artifact_download | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| target: aarch64-unknown-linux-gnu | |
| path: ./distro/linux/arm64/ | |
| category: udf | |
| artifacts: query | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend-query | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| - name: Prepare for docker | |
| id: prepare | |
| run: | | |
| short_sha=$(echo "${{ needs.info.outputs.sha }}" | cut -c1-7) | |
| ts=$(date +%s) | |
| echo "tag=pr-${{ github.event.pull_request.number }}-${short_sha}-${ts}" >> $GITHUB_OUTPUT | |
| - name: push service image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| push: true | |
| tags: ${{ steps.login.outputs.ecr_repo }}:${{ steps.prepare.outputs.tag }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/service/query.Dockerfile | |
| - name: Comment on PR | |
| uses: everpcpc/comment-on-pr-action@v1 | |
| with: | |
| number: ${{ github.event.pull_request.number }} | |
| token: ${{ github.token }} | |
| body: | | |
| ## Docker Image for PR | |
| * **tag**: `${{ steps.prepare.outputs.tag }}` | |
| > note: this image tag is only available for internal use, | |
| > please check the [internal doc](https://datafuselabs.feishu.cn/wiki/Hk7iwYsnWi3wsXkpbW5c9apTn0b) for more details. | |
| benchmark: | |
| if: needs.info.outputs.target | |
| needs: [info, build, docker] | |
| uses: ./.github/workflows/reuse.benchmark.yml | |
| secrets: inherit | |
| with: | |
| sha: ${{ needs.info.outputs.sha }} | |
| run_id: ${{ github.run_id }} | |
| source: pr | |
| source_id: ${{ github.event.pull_request.number }} | |
| version: ${{ needs.docker.outputs.tag }} | |
| runner_provider: aws | |
| target: ${{ needs.info.outputs.target }} |