chore: patch z3-sys & add musl build to ci (#14786)

databendlabs · Feb 29, 2024 · 640a044 · 640a044
1 parent c54890d
commit 640a044
Show file tree

Hide file tree

Showing 15 changed files with 180 additions and 134 deletions.
diff --git a/.github/actions/build_bindings_python/action.yml b/.github/actions/build_bindings_python/action.yml
@@ -51,7 +51,7 @@ runs:
         rust-toolchain: ${{ steps.toolchain.outputs.RUST_TOOLCHAIN }}
         working-directory: src/bendpy
         target: ${{ inputs.target }}
-        manylinux: auto
+        manylinux: "2_28"
         # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
         rustup-components: rust-std rustfmt
         args: ${{ steps.opts.outputs.BUILD_ARGS }}
@@ -62,7 +62,7 @@ runs:
 
     - name: Run tests
       working-directory: src/bendpy
-      if: startsWith(inputs.target, 'x86_64-')
+      if: endsWith(inputs.target, '-gnu')
       shell: bash
       run: |
         pip install dist/*.whl

diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml
@@ -46,13 +46,13 @@ runs:
             flags="-C target-feature=+sse4.2"
             ;;
           x86_64-unknown-linux-musl)
-            flags="-C target-feature=+sse4.2 -C link-arg=--max-memory=4294967296"
+            flags="-C target-feature=+sse4.2"
             ;;
           aarch64-unknown-linux-gnu)
             flags=""
             ;;
           aarch64-unknown-linux-musl)
-            flags="-C link-arg=--max-memory=4294967296"
+            flags=""
             ;;
           *)
             flags=""

diff --git a/.github/actions/build_linux_sanitizer/action.yml b/.github/actions/build_linux_sanitizer/action.yml
@@ -50,7 +50,7 @@ runs:
       if: env.BUILD_PROFILE == 'debug' && endsWith(inputs.target, '-gnu')
       shell: bash
       run: |
-        artifacts="meta,metactl,query"
+        artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact with sanitizer ..."
           cargo -Zbuild-std -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --manifest-path src/binaries/Cargo.toml --bin databend-$artifact

diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml
@@ -20,18 +20,19 @@ concurrency:
 
 jobs:
   linux:
-    runs-on: [self-hosted, X64, Linux, 8c16g, gcp]
+    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c16g, gcp]
     strategy:
       matrix:
-        target:
-          - x86_64-unknown-linux-gnu
+        include:
+          - { arch: x86_64, runner: X64 }
+          # - { arch: aarch64, runner: ARM64 }
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ./.github/actions/build_bindings_python
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-unknown-linux-gnu
           version: ${{ inputs.tag }}
       - name: upload
         if: inputs.tag
@@ -45,15 +46,15 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        target:
-          - aarch64-apple-darwin
+        arch:
+          - aarch64
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ./.github/actions/build_bindings_python
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-apple-darwin
           version: ${{ inputs.tag }}
       - name: upload
         if: inputs.tag

diff --git a/.github/workflows/build_tool.yml b/.github/workflows/build_tool.yml
@@ -13,72 +13,94 @@ permissions:
   contents: read
 
 jobs:
-  build:
+  debian:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        os:
-          - debian
-          - alpine
     steps:
       - uses: actions/checkout@v4
-
       - uses: ./.github/actions/setup_docker
         id: login
         with:
           repo: build-tool
           ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
           dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Get rust toolchain version
         id: toolchain
         run: |
           version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
           echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
-
       - name: Build and publish databend build image
         uses: docker/build-push-action@v5
         with:
           pull: true
           push: true
           tags: |
-            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.os }}
-            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.os }}-${{ steps.toolchain.outputs.TOOLCHAIN }}
+            ${{ steps.login.outputs.dockerhub_repo }}:debian
+            ${{ steps.login.outputs.dockerhub_repo }}:debian-${{ steps.toolchain.outputs.TOOLCHAIN }}
           context: ./scripts/setup
-          file: ./docker/build-tool/${{ matrix.os }}/Dockerfile
+          file: ./docker/build-tool/debian/Dockerfile
           platforms: linux/amd64,linux/arm64
 
-  dev:
+  musl:
     runs-on: ubuntu-latest
-    needs: build
+    strategy:
+      matrix:
+        arch:
+          - aarch64
+          - x86_64
     steps:
       - uses: actions/checkout@v4
-
       - uses: ./.github/actions/setup_docker
         id: login
         with:
           repo: build-tool
           ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
           dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Get rust toolchain version
         id: toolchain
         run: |
           version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
           echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
+      - name: Build and publish databend build image
+        uses: docker/build-push-action@v5
+        with:
+          pull: true
+          push: true
+          tags: |
+            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.arch }}-unknown-linux-musl
+            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.arch }}-unknown-linux-musl-${{ steps.toolchain.outputs.TOOLCHAIN }}
+          context: ./docker/build-tool/musl
+          file: ./docker/build-tool/musl/Dockerfile
+          build-args: |
+            ARCH=${{ matrix.arch }}
+            RUST_TOOLCHAIN=${{ steps.toolchain.outputs.TOOLCHAIN }}
 
-      - name: Build and publish databend build base image
+  dev:
+    runs-on: ubuntu-latest
+    needs: debian
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup_docker
+        id: login
+        with:
+          repo: build-tool
+          ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
+          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Get rust toolchain version
+        id: toolchain
+        run: |
+          version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
+          echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
+      - name: Build and publish databend build image
         uses: docker/build-push-action@v5
         with:
           pull: true
           push: true
           tags: |
             ${{ steps.login.outputs.dockerhub_repo }}:dev
             ${{ steps.login.outputs.dockerhub_repo }}:dev-${{ steps.toolchain.outputs.TOOLCHAIN }}
-            ${{ steps.login.outputs.ecr_repo }}:dev
-            ${{ steps.login.outputs.ecr_repo }}:dev-${{ steps.toolchain.outputs.TOOLCHAIN }}
           context: ./scripts/setup
           file: ./docker/build-tool/dev/Dockerfile
           build-args: |

diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml
@@ -47,8 +47,8 @@ jobs:
     strategy:
       matrix:
         include:
-          - {target: x86_64-unknown-linux-gnu, runner: X64}
-          - {target: aarch64-unknown-linux-gnu, runner: ARM64}
+          - {arch: x86_64, runner: X64}
+          - {arch: aarch64, runner: ARM64}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -59,7 +59,7 @@ jobs:
         timeout-minutes: 60
         with:
           sha: ${{ needs.info.outputs.sha }}
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-unknown-linux-gnu
           artifacts: meta,query
 
   docker:

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -59,7 +59,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { target: x86_64-unknown-linux-gnu, runner: X64 }
+          - { arch: x86_64, runner: X64 }
     steps:
       - uses: actions/checkout@v4
         with:
@@ -69,7 +69,7 @@ jobs:
         timeout-minutes: 60
         with:
           sha: ${{ github.sha }}
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-unknown-linux-gnu
           artifacts: all
 
   build_address_sanitizer:
@@ -78,7 +78,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { target: aarch64-unknown-linux-gnu, runner: ARM64 }
+          - { arch: aarch64, runner: ARM64 }
     steps:
       - uses: actions/checkout@v4
         with:
@@ -87,27 +87,28 @@ jobs:
       - uses: ./.github/actions/build_linux_sanitizer
         timeout-minutes: 60
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-unknown-linux-gnu
           artifacts: query
 
-  # build_other:
-  #   runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"]
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       include:
-  #         - { target: aarch64-unknown-linux-musl, runner: ARM64 }
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #       with:
-  #         # fetch all tags, metasrv and metaclient need tag as its version.
-  #         fetch-depth: 0
-  #     - uses: ./.github/actions/build_linux
-  #       timeout-minutes: 60
-  #       with:
-  #         sha: ${{ github.sha }}
-  #         target: ${{ matrix.target }}
-  #         artifacts: query
+  build_musl:
+    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x86_64
+          - aarch64
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # fetch all tags, metasrv and metaclient need tag as its version.
+          fetch-depth: 0
+      - uses: ./.github/actions/build_linux
+        timeout-minutes: 60
+        with:
+          sha: ${{ github.sha }}
+          target: ${{ matrix.arch }}-unknown-linux-musl
+          artifacts: query
 
   test_unit:
     runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]