From 352916dac7081060f38f6efe4f31215fdd952a5a Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 9 Sep 2024 15:06:03 +0300 Subject: [PATCH 1/3] add multi-platform support to buildx Signed-off-by: Oguz Ozturk --- .../actions/docker-build-and-push/action.yaml | 411 +++++++++--------- .github/workflows/docker-build-and-push.yaml | 166 +++---- 2 files changed, 291 insertions(+), 286 deletions(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 4c9ab20834..bb1c043778 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -1,203 +1,208 @@ -name: docker-build-and-push -description: "" - -inputs: - name: - description: "" - required: true - platform: - description: "" - required: true - bake-target: - description: "" - required: true - build-args: - description: "" - required: false - tag-prefix: - description: "" - required: false - tag-suffix: - description: "" - required: false - -runs: - using: composite - steps: - - name: Install jq and vcstool - run: | - sudo apt-get -y update - sudo apt-get -y install jq python3-pip - pip install --no-cache-dir vcstool - shell: bash - - - name: Run vcs import - run: | - mkdir src - vcs import src < autoware.repos - shell: bash - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Restore ccache - uses: actions/cache/restore@v4 - with: - path: | - root-ccache - key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} - restore-keys: | - ccache-${{ inputs.platform }}-${{ inputs.name }}- - ccache-${{ inputs.platform }}- - - - name: Restore apt-get - uses: actions/cache/restore@v4 - with: - path: | - var-cache-apt - key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} - restore-keys: | - apt-get-${{ inputs.platform }}-${{ inputs.name }}- - apt-get-${{ inputs.platform }}- - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "root-ccache": "/root/.ccache", - "var-cache-apt": "/var/cache/apt" - } - skip-extraction: true - - - name: Set Docker tags - id: set-docker-tags - run: | - tags=() - if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then - tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") - fi - - tags+=("{{date 'YYYYMMDD'}}") - tags+=("latest") - tags+=("latest-${{ inputs.tag-prefix }}") - - # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "tags<<$EOF" >> $GITHUB_OUTPUT - echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - shell: bash - - - name: Docker meta for autoware:base - id: meta-base - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-base - flavor: | - latest=false - suffix=-base${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:core-devel - id: meta-core-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-core-devel - flavor: | - latest=false - suffix=-core-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-sensing-perception-devel - id: meta-universe-sensing-perception-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-sensing-perception-devel - flavor: | - latest=false - suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-sensing-perception - id: meta-universe-sensing-perception - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-sensing-perception - flavor: | - latest=false - suffix=-universe-sensing-perception${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-localization-mapping-devel - id: meta-universe-localization-mapping-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-localization-mapping-devel - flavor: | - latest=false - suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-localization-mapping - id: meta-universe-localization-mapping - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-localization-mapping - flavor: | - latest=false - suffix=-universe-localization-mapping${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-devel - id: meta-universe-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-devel - flavor: | - latest=false - suffix=-universe-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe - id: meta-universe - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe - flavor: | - latest=auto - suffix=-universe${{ inputs.tag-suffix }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Build and Push to GitHub Container Registry - uses: docker/bake-action@v5 - with: - push: true - files: | - docker/docker-bake.hcl - ${{ steps.meta-base.outputs.bake-file }} - ${{ steps.meta-core-devel.outputs.bake-file }} - ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} - ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} - ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} - ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} - ${{ steps.meta-universe-devel.outputs.bake-file }} - ${{ steps.meta-universe.outputs.bake-file }} - provenance: false - set: | - ${{ inputs.build-args }} +name: docker-build-and-push +description: "" + +inputs: + name: + description: "" + required: true + platform: + description: "" + required: true + bake-target: + description: "" + required: true + build-args: + description: "" + required: false + tag-prefix: + description: "" + required: false + tag-suffix: + description: "" + required: false + +runs: + using: composite + steps: + - name: Install jq and vcstool + run: | + sudo apt-get -y update + sudo apt-get -y install jq python3-pip + pip install --no-cache-dir vcstool + shell: bash + + - name: Run vcs import + run: | + mkdir src + vcs import src < autoware.repos + shell: bash + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: | + root-ccache + key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} + restore-keys: | + ccache-${{ inputs.platform }}-${{ inputs.name }}- + ccache-${{ inputs.platform }}- + + - name: Restore apt-get + uses: actions/cache/restore@v4 + with: + path: | + var-cache-apt + key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} + restore-keys: | + apt-get-${{ inputs.platform }}-${{ inputs.name }}- + apt-get-${{ inputs.platform }}- + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "root-ccache": "/root/.ccache", + "var-cache-apt": "/var/cache/apt" + } + skip-extraction: true + + - name: Set Docker tags + id: set-docker-tags + run: | + tags=() + if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then + tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") + fi + + tags+=("{{date 'YYYYMMDD'}}") + tags+=("latest") + tags+=("latest-${{ inputs.tag-prefix }}") + + # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "tags<<$EOF" >> $GITHUB_OUTPUT + echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + shell: bash + + - name: Docker meta for autoware:base + id: meta-base + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-base + flavor: | + latest=false + suffix=-base${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:core-devel + id: meta-core-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-core-devel + flavor: | + latest=false + suffix=-core-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception-devel + id: meta-universe-sensing-perception-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception-devel + flavor: | + latest=false + suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception + id: meta-universe-sensing-perception + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception + flavor: | + latest=false + suffix=-universe-sensing-perception${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping-devel + id: meta-universe-localization-mapping-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping-devel + flavor: | + latest=false + suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping + id: meta-universe-localization-mapping + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping + flavor: | + latest=false + suffix=-universe-localization-mapping${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-devel + id: meta-universe-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-devel + flavor: | + latest=false + suffix=-universe-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe + id: meta-universe + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe + flavor: | + latest=auto + suffix=-universe${{ inputs.tag-suffix }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Build and Push to GitHub Container Registry + uses: docker/bake-action@v5 + with: + push: true + files: | + docker/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} + ${{ steps.meta-core-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} + ${{ steps.meta-universe-devel.outputs.bake-file }} + ${{ steps.meta-universe.outputs.bake-file }} + provenance: false + set: | + ${{ inputs.build-args }} diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml index 9199634b8c..9cb90dc66c 100644 --- a/.github/workflows/docker-build-and-push.yaml +++ b/.github/workflows/docker-build-and-push.yaml @@ -1,83 +1,83 @@ -name: docker-build-and-push - -on: - push: - branches: - - main - tags: - workflow_dispatch: - -jobs: - load-env: - uses: ./.github/workflows/load-env.yaml - - docker-build-and-push: - needs: load-env - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - name: - - no-cuda - - cuda - include: - - name: no-cuda - platform: amd64 - base_image_env: base_image - lib_dir: x86_64 - setup-args: --no-nvidia - additional-tag-suffix: "" - - name: cuda - platform: amd64 - base_image_env: base_image - lib_dir: x86_64 - additional-tag-suffix: -cuda - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set git config - uses: autowarefoundation/autoware-github-actions/set-git-config@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Free disk space - uses: ./.github/actions/free-disk-space - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - *.env - *.repos - .github/actions/docker-build-and-push/action.yaml - .github/workflows/docker-build-and-push*.yaml - ansible-galaxy-requirements.yaml - ansible/** - docker/** - - - name: Build 'Autoware' - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref_type == 'tag') }} - uses: ./.github/actions/docker-build-and-push - with: - name: ${{ matrix.name }} - platform: ${{ matrix.platform }} - bake-target: autoware - build-args: | - *.platform=linux/${{ matrix.platform }} - *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} - *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} - *.args.SETUP_ARGS=${{ matrix.setup-args }} - *.args.LIB_DIR=${{ matrix.lib_dir }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main - *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max - tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} - tag-prefix: ${{ needs.load-env.outputs.rosdistro }} - - - name: Show disk space - run: | - df -h +name: docker-build-and-push + +on: + push: + branches: + - main + tags: + workflow_dispatch: + +jobs: + load-env: + uses: ./.github/workflows/load-env.yaml + + docker-build-and-push: + needs: load-env + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: + - no-cuda + - cuda + include: + - name: no-cuda + platform: linux/amd64,linux/arm64 + base_image_env: base_image + lib_dir: x86_64 + setup-args: --no-nvidia + additional-tag-suffix: "" + - name: cuda + platform: linux/amd64,linux/arm64 + base_image_env: base_image + lib_dir: x86_64 + additional-tag-suffix: -cuda + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + *.env + *.repos + .github/actions/docker-build-and-push/action.yaml + .github/workflows/docker-build-and-push*.yaml + ansible-galaxy-requirements.yaml + ansible/** + docker/** + + - name: Build 'Autoware' + if: ${{ steps.changed-files.outputs.any_changed == 'true' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && github.ref_type == 'tag') }} + uses: ./.github/actions/docker-build-and-push + with: + name: ${{ matrix.name }} + platform: ${{ matrix.platform }} + bake-target: autoware + build-args: | + *.platform=${{ matrix.platform }} + *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} + *.args.SETUP_ARGS=${{ matrix.setup-args }} + *.args.LIB_DIR=${{ matrix.lib_dir }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main + *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max + tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} + tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + + - name: Show disk space + run: | + df -h From bec9506eca55d1f9d42746e9cda13ad9e127d387 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 9 Sep 2024 16:47:55 +0300 Subject: [PATCH 2/3] Revert "add multi-platform support to buildx" This reverts commit 352916dac7081060f38f6efe4f31215fdd952a5a. --- .../actions/docker-build-and-push/action.yaml | 411 +++++++++--------- .github/workflows/docker-build-and-push.yaml | 166 +++---- 2 files changed, 286 insertions(+), 291 deletions(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index bb1c043778..4c9ab20834 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -1,208 +1,203 @@ -name: docker-build-and-push -description: "" - -inputs: - name: - description: "" - required: true - platform: - description: "" - required: true - bake-target: - description: "" - required: true - build-args: - description: "" - required: false - tag-prefix: - description: "" - required: false - tag-suffix: - description: "" - required: false - -runs: - using: composite - steps: - - name: Install jq and vcstool - run: | - sudo apt-get -y update - sudo apt-get -y install jq python3-pip - pip install --no-cache-dir vcstool - shell: bash - - - name: Run vcs import - run: | - mkdir src - vcs import src < autoware.repos - shell: bash - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - platforms: linux/amd64,linux/arm64 - - - name: Restore ccache - uses: actions/cache/restore@v4 - with: - path: | - root-ccache - key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} - restore-keys: | - ccache-${{ inputs.platform }}-${{ inputs.name }}- - ccache-${{ inputs.platform }}- - - - name: Restore apt-get - uses: actions/cache/restore@v4 - with: - path: | - var-cache-apt - key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} - restore-keys: | - apt-get-${{ inputs.platform }}-${{ inputs.name }}- - apt-get-${{ inputs.platform }}- - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "root-ccache": "/root/.ccache", - "var-cache-apt": "/var/cache/apt" - } - skip-extraction: true - - - name: Set Docker tags - id: set-docker-tags - run: | - tags=() - if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then - tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") - fi - - tags+=("{{date 'YYYYMMDD'}}") - tags+=("latest") - tags+=("latest-${{ inputs.tag-prefix }}") - - # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "tags<<$EOF" >> $GITHUB_OUTPUT - echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - shell: bash - - - name: Docker meta for autoware:base - id: meta-base - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-base - flavor: | - latest=false - suffix=-base${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:core-devel - id: meta-core-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-core-devel - flavor: | - latest=false - suffix=-core-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-sensing-perception-devel - id: meta-universe-sensing-perception-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-sensing-perception-devel - flavor: | - latest=false - suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-sensing-perception - id: meta-universe-sensing-perception - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-sensing-perception - flavor: | - latest=false - suffix=-universe-sensing-perception${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-localization-mapping-devel - id: meta-universe-localization-mapping-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-localization-mapping-devel - flavor: | - latest=false - suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-localization-mapping - id: meta-universe-localization-mapping - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-localization-mapping - flavor: | - latest=false - suffix=-universe-localization-mapping${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-devel - id: meta-universe-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-devel - flavor: | - latest=false - suffix=-universe-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe - id: meta-universe - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe - flavor: | - latest=auto - suffix=-universe${{ inputs.tag-suffix }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Build and Push to GitHub Container Registry - uses: docker/bake-action@v5 - with: - push: true - files: | - docker/docker-bake.hcl - ${{ steps.meta-base.outputs.bake-file }} - ${{ steps.meta-core-devel.outputs.bake-file }} - ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} - ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} - ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} - ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} - ${{ steps.meta-universe-devel.outputs.bake-file }} - ${{ steps.meta-universe.outputs.bake-file }} - provenance: false - set: | - ${{ inputs.build-args }} +name: docker-build-and-push +description: "" + +inputs: + name: + description: "" + required: true + platform: + description: "" + required: true + bake-target: + description: "" + required: true + build-args: + description: "" + required: false + tag-prefix: + description: "" + required: false + tag-suffix: + description: "" + required: false + +runs: + using: composite + steps: + - name: Install jq and vcstool + run: | + sudo apt-get -y update + sudo apt-get -y install jq python3-pip + pip install --no-cache-dir vcstool + shell: bash + + - name: Run vcs import + run: | + mkdir src + vcs import src < autoware.repos + shell: bash + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: | + root-ccache + key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} + restore-keys: | + ccache-${{ inputs.platform }}-${{ inputs.name }}- + ccache-${{ inputs.platform }}- + + - name: Restore apt-get + uses: actions/cache/restore@v4 + with: + path: | + var-cache-apt + key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} + restore-keys: | + apt-get-${{ inputs.platform }}-${{ inputs.name }}- + apt-get-${{ inputs.platform }}- + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "root-ccache": "/root/.ccache", + "var-cache-apt": "/var/cache/apt" + } + skip-extraction: true + + - name: Set Docker tags + id: set-docker-tags + run: | + tags=() + if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then + tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") + fi + + tags+=("{{date 'YYYYMMDD'}}") + tags+=("latest") + tags+=("latest-${{ inputs.tag-prefix }}") + + # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "tags<<$EOF" >> $GITHUB_OUTPUT + echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + shell: bash + + - name: Docker meta for autoware:base + id: meta-base + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-base + flavor: | + latest=false + suffix=-base${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:core-devel + id: meta-core-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-core-devel + flavor: | + latest=false + suffix=-core-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception-devel + id: meta-universe-sensing-perception-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception-devel + flavor: | + latest=false + suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception + id: meta-universe-sensing-perception + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception + flavor: | + latest=false + suffix=-universe-sensing-perception${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping-devel + id: meta-universe-localization-mapping-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping-devel + flavor: | + latest=false + suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping + id: meta-universe-localization-mapping + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping + flavor: | + latest=false + suffix=-universe-localization-mapping${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-devel + id: meta-universe-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-devel + flavor: | + latest=false + suffix=-universe-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe + id: meta-universe + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe + flavor: | + latest=auto + suffix=-universe${{ inputs.tag-suffix }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Build and Push to GitHub Container Registry + uses: docker/bake-action@v5 + with: + push: true + files: | + docker/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} + ${{ steps.meta-core-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} + ${{ steps.meta-universe-devel.outputs.bake-file }} + ${{ steps.meta-universe.outputs.bake-file }} + provenance: false + set: | + ${{ inputs.build-args }} diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml index 9cb90dc66c..9199634b8c 100644 --- a/.github/workflows/docker-build-and-push.yaml +++ b/.github/workflows/docker-build-and-push.yaml @@ -1,83 +1,83 @@ -name: docker-build-and-push - -on: - push: - branches: - - main - tags: - workflow_dispatch: - -jobs: - load-env: - uses: ./.github/workflows/load-env.yaml - - docker-build-and-push: - needs: load-env - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - name: - - no-cuda - - cuda - include: - - name: no-cuda - platform: linux/amd64,linux/arm64 - base_image_env: base_image - lib_dir: x86_64 - setup-args: --no-nvidia - additional-tag-suffix: "" - - name: cuda - platform: linux/amd64,linux/arm64 - base_image_env: base_image - lib_dir: x86_64 - additional-tag-suffix: -cuda - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set git config - uses: autowarefoundation/autoware-github-actions/set-git-config@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Free disk space - uses: ./.github/actions/free-disk-space - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - *.env - *.repos - .github/actions/docker-build-and-push/action.yaml - .github/workflows/docker-build-and-push*.yaml - ansible-galaxy-requirements.yaml - ansible/** - docker/** - - - name: Build 'Autoware' - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref_type == 'tag') }} - uses: ./.github/actions/docker-build-and-push - with: - name: ${{ matrix.name }} - platform: ${{ matrix.platform }} - bake-target: autoware - build-args: | - *.platform=${{ matrix.platform }} - *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} - *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} - *.args.SETUP_ARGS=${{ matrix.setup-args }} - *.args.LIB_DIR=${{ matrix.lib_dir }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main - *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max - tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} - tag-prefix: ${{ needs.load-env.outputs.rosdistro }} - - - name: Show disk space - run: | - df -h +name: docker-build-and-push + +on: + push: + branches: + - main + tags: + workflow_dispatch: + +jobs: + load-env: + uses: ./.github/workflows/load-env.yaml + + docker-build-and-push: + needs: load-env + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: + - no-cuda + - cuda + include: + - name: no-cuda + platform: amd64 + base_image_env: base_image + lib_dir: x86_64 + setup-args: --no-nvidia + additional-tag-suffix: "" + - name: cuda + platform: amd64 + base_image_env: base_image + lib_dir: x86_64 + additional-tag-suffix: -cuda + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + *.env + *.repos + .github/actions/docker-build-and-push/action.yaml + .github/workflows/docker-build-and-push*.yaml + ansible-galaxy-requirements.yaml + ansible/** + docker/** + + - name: Build 'Autoware' + if: ${{ steps.changed-files.outputs.any_changed == 'true' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && github.ref_type == 'tag') }} + uses: ./.github/actions/docker-build-and-push + with: + name: ${{ matrix.name }} + platform: ${{ matrix.platform }} + bake-target: autoware + build-args: | + *.platform=linux/${{ matrix.platform }} + *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} + *.args.SETUP_ARGS=${{ matrix.setup-args }} + *.args.LIB_DIR=${{ matrix.lib_dir }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main + *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max + tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} + tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + + - name: Show disk space + run: | + df -h From 8d2eb4dc652bc6738c8c3e300cb20477c793937e Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 9 Sep 2024 18:02:11 +0300 Subject: [PATCH 3/3] add cross-compilation with qemu Signed-off-by: Oguz Ozturk --- .../actions/docker-build-and-push/action.yaml | 407 +++++++++--------- .github/actions/docker-build/action.yaml | 214 ++++----- .github/actions/docker-buildv2/action.yaml | 204 +++++++++ .../docker-build-and-push-arm64.yaml | 176 ++++---- .github/workflows/docker-build-and-push.yaml | 209 +++++---- 5 files changed, 729 insertions(+), 481 deletions(-) create mode 100644 .github/actions/docker-buildv2/action.yaml diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 4c9ab20834..be03433f60 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -1,203 +1,204 @@ -name: docker-build-and-push -description: "" - -inputs: - name: - description: "" - required: true - platform: - description: "" - required: true - bake-target: - description: "" - required: true - build-args: - description: "" - required: false - tag-prefix: - description: "" - required: false - tag-suffix: - description: "" - required: false - -runs: - using: composite - steps: - - name: Install jq and vcstool - run: | - sudo apt-get -y update - sudo apt-get -y install jq python3-pip - pip install --no-cache-dir vcstool - shell: bash - - - name: Run vcs import - run: | - mkdir src - vcs import src < autoware.repos - shell: bash - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Restore ccache - uses: actions/cache/restore@v4 - with: - path: | - root-ccache - key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} - restore-keys: | - ccache-${{ inputs.platform }}-${{ inputs.name }}- - ccache-${{ inputs.platform }}- - - - name: Restore apt-get - uses: actions/cache/restore@v4 - with: - path: | - var-cache-apt - key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} - restore-keys: | - apt-get-${{ inputs.platform }}-${{ inputs.name }}- - apt-get-${{ inputs.platform }}- - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "root-ccache": "/root/.ccache", - "var-cache-apt": "/var/cache/apt" - } - skip-extraction: true - - - name: Set Docker tags - id: set-docker-tags - run: | - tags=() - if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then - tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") - fi - - tags+=("{{date 'YYYYMMDD'}}") - tags+=("latest") - tags+=("latest-${{ inputs.tag-prefix }}") - - # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "tags<<$EOF" >> $GITHUB_OUTPUT - echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - shell: bash - - - name: Docker meta for autoware:base - id: meta-base - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-base - flavor: | - latest=false - suffix=-base${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:core-devel - id: meta-core-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-core-devel - flavor: | - latest=false - suffix=-core-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-sensing-perception-devel - id: meta-universe-sensing-perception-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-sensing-perception-devel - flavor: | - latest=false - suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-sensing-perception - id: meta-universe-sensing-perception - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-sensing-perception - flavor: | - latest=false - suffix=-universe-sensing-perception${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-localization-mapping-devel - id: meta-universe-localization-mapping-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-localization-mapping-devel - flavor: | - latest=false - suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-localization-mapping - id: meta-universe-localization-mapping - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-localization-mapping - flavor: | - latest=false - suffix=-universe-localization-mapping${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe-devel - id: meta-universe-devel - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe-devel - flavor: | - latest=false - suffix=-universe-devel${{ inputs.tag-suffix }} - - - name: Docker meta for autoware:universe - id: meta-universe - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-universe - flavor: | - latest=auto - suffix=-universe${{ inputs.tag-suffix }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Build and Push to GitHub Container Registry - uses: docker/bake-action@v5 - with: - push: true - files: | - docker/docker-bake.hcl - ${{ steps.meta-base.outputs.bake-file }} - ${{ steps.meta-core-devel.outputs.bake-file }} - ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} - ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} - ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} - ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} - ${{ steps.meta-universe-devel.outputs.bake-file }} - ${{ steps.meta-universe.outputs.bake-file }} - provenance: false - set: | - ${{ inputs.build-args }} +name: docker-build-and-push +description: "" + +inputs: + name: + description: "" + required: true + platform: + description: "" + required: true + bake-target: + description: "" + required: true + build-args: + description: "" + required: false + tag-prefix: + description: "" + required: false + tag-suffix: + description: "" + required: false + +runs: + using: composite + steps: + - name: Install jq and vcstool + run: | + sudo apt-get -y update + sudo apt-get -y install jq python3-pip + pip install --no-cache-dir vcstool + shell: bash + + - name: Run vcs import + run: | + mkdir src + vcs import src < autoware.repos + shell: bash + + - name: Setup QEMU for cross-compilation + if: ${{ inputs.platform == 'arm64' }} + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ inputs.platform }} + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: ${{ inputs.platform }} + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: | + root-ccache + key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} + restore-keys: | + ccache-${{ inputs.platform }}-${{ inputs.name }}- + ccache-${{ inputs.platform }}- + + - name: Restore apt-get + uses: actions/cache/restore@v4 + with: + path: | + var-cache-apt + key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} + restore-keys: | + apt-get-${{ inputs.platform }}-${{ inputs.name }}- + apt-get-${{ inputs.platform }}- + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "root-ccache": "/root/.ccache", + "var-cache-apt": "/var/cache/apt" + } + skip-extraction: true + + - name: Set Docker tags + id: set-docker-tags + run: | + tags=() + if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then + tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") + fi + + tags+=("{{date 'YYYYMMDD'}}") + tags+=("latest") + tags+=("latest-${{ inputs.tag-prefix }}") + + # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "tags<<$EOF" >> $GITHUB_OUTPUT + echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + shell: bash + + - name: Docker meta for autoware:base + id: meta-base + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-base + flavor: | + latest=false + suffix=-base${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:core-devel + id: meta-core-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-core-devel + flavor: | + latest=false + suffix=-core-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception-devel + id: meta-universe-sensing-perception-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception-devel + flavor: | + latest=false + suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception + id: meta-universe-sensing-perception + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception + flavor: | + latest=false + suffix=-universe-sensing-perception${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping-devel + id: meta-universe-localization-mapping-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping-devel + flavor: | + latest=false + suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping + id: meta-universe-localization-mapping + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping + flavor: | + latest=false + suffix=-universe-localization-mapping${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-devel + id: meta-universe-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-devel + flavor: | + latest=false + suffix=-universe-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe + id: meta-universe + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe + flavor: | + latest=auto + suffix=-universe${{ inputs.tag-suffix }} + + - name: Build and Push to GitHub Container Registry + uses: docker/bake-action@v5 + with: + push: false + files: | + docker/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} + ${{ steps.meta-core-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} + ${{ steps.meta-universe-devel.outputs.bake-file }} + ${{ steps.meta-universe.outputs.bake-file }} + provenance: false + set: | + ${{ inputs.build-args }} diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index de58261188..4e4d4dfe1f 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -1,107 +1,107 @@ -name: docker-build -description: "" - -inputs: - name: - description: "" - required: true - platform: - description: "" - required: true - cache-tag-suffix: - description: "" - required: true - build-args: - description: "" - required: true - -runs: - using: composite - steps: - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Install vcstool - run: | - sudo apt-get -y update - sudo apt-get -y install python3-pip - pip install --no-cache-dir vcstool - shell: bash - - - name: Run vcs import - run: | - mkdir src - vcs import src < autoware.repos - shell: bash - - - name: Cache ccache - uses: actions/cache@v4 - if: ${{ inputs.name == 'no-cuda' && github.ref == 'refs/heads/main' }} - id: cache-ccache - with: - path: | - root-ccache - key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} - restore-keys: | - ccache-${{ inputs.platform }}-${{ inputs.name }}- - ccache-${{ inputs.platform }}- - - - name: Cache apt-get - uses: actions/cache@v4 - if: ${{ inputs.name == 'no-cuda' && github.ref == 'refs/heads/main' }} - id: cache-apt-get - with: - path: | - var-cache-apt - key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} - restore-keys: | - apt-get-${{ inputs.platform }}-${{ inputs.name }}- - apt-get-${{ inputs.platform }}- - - - name: Restore ccache - uses: actions/cache/restore@v4 - if: ${{ inputs.name != 'no-cuda' || github.ref != 'refs/heads/main' }} - with: - path: | - root-ccache - key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} - restore-keys: | - ccache-${{ inputs.platform }}-${{ inputs.name }}- - ccache-${{ inputs.platform }}- - - - name: Restore apt-get - uses: actions/cache/restore@v4 - if: ${{ inputs.name != 'no-cuda' || github.ref != 'refs/heads/main' }} - with: - path: | - var-cache-apt - key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} - restore-keys: | - apt-get-${{ inputs.platform }}-${{ inputs.name }}- - apt-get-${{ inputs.platform }}- - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "root-ccache": "/root/.ccache", - "var-cache-apt": "/var/cache/apt" - } - skip-extraction: ${{ steps.cache-ccache.outputs.cache-hit && steps.cache-apt-get.outputs.cache-hit }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Run docker build - uses: docker/build-push-action@v6 - with: - file: docker/Dockerfile - context: . - push: false - build-args: ${{ inputs.build-args }} - cache-from: type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ inputs.name }}-${{ inputs.platform }}-${{ inputs.cache-tag-suffix }} +name: docker-build +description: "" + +inputs: + name: + description: "" + required: true + platform: + description: "" + required: true + cache-tag-suffix: + description: "" + required: true + build-args: + description: "" + required: true + +runs: + using: composite + steps: + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install vcstool + run: | + sudo apt-get -y update + sudo apt-get -y install python3-pip + pip install --no-cache-dir vcstool + shell: bash + + - name: Run vcs import + run: | + mkdir src + vcs import src < autoware.repos + shell: bash + + - name: Cache ccache + uses: actions/cache@v4 + if: ${{ inputs.name == 'no-cuda' && github.ref == 'refs/heads/main' }} + id: cache-ccache + with: + path: | + root-ccache + key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} + restore-keys: | + ccache-${{ inputs.platform }}-${{ inputs.name }}- + ccache-${{ inputs.platform }}- + + - name: Cache apt-get + uses: actions/cache@v4 + if: ${{ inputs.name == 'no-cuda' && github.ref == 'refs/heads/main' }} + id: cache-apt-get + with: + path: | + var-cache-apt + key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} + restore-keys: | + apt-get-${{ inputs.platform }}-${{ inputs.name }}- + apt-get-${{ inputs.platform }}- + + - name: Restore ccache + uses: actions/cache/restore@v4 + if: ${{ inputs.name != 'no-cuda' || github.ref != 'refs/heads/main' }} + with: + path: | + root-ccache + key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} + restore-keys: | + ccache-${{ inputs.platform }}-${{ inputs.name }}- + ccache-${{ inputs.platform }}- + + - name: Restore apt-get + uses: actions/cache/restore@v4 + if: ${{ inputs.name != 'no-cuda' || github.ref != 'refs/heads/main' }} + with: + path: | + var-cache-apt + key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} + restore-keys: | + apt-get-${{ inputs.platform }}-${{ inputs.name }}- + apt-get-${{ inputs.platform }}- + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "root-ccache": "/root/.ccache", + "var-cache-apt": "/var/cache/apt" + } + skip-extraction: ${{ steps.cache-ccache.outputs.cache-hit && steps.cache-apt-get.outputs.cache-hit }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Run docker build + uses: docker/build-push-action@v6 + with: + file: docker/Dockerfile + context: . + push: false + build-args: ${{ inputs.build-args }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ inputs.name }}-${{ inputs.platform }}-${{ inputs.cache-tag-suffix }} diff --git a/.github/actions/docker-buildv2/action.yaml b/.github/actions/docker-buildv2/action.yaml new file mode 100644 index 0000000000..eaac18a081 --- /dev/null +++ b/.github/actions/docker-buildv2/action.yaml @@ -0,0 +1,204 @@ +name: docker-buildv2 +description: "" + +inputs: + name: + description: "" + required: true + platform: + description: "" + required: true + bake-target: + description: "" + required: true + build-args: + description: "" + required: false + tag-prefix: + description: "" + required: false + tag-suffix: + description: "" + required: false + +runs: + using: composite + steps: + - name: Install jq and vcstool + run: | + sudo apt-get -y update + sudo apt-get -y install jq python3-pip + pip install --no-cache-dir vcstool + shell: bash + + - name: Run vcs import + run: | + mkdir src + vcs import src < autoware.repos + shell: bash + + - name: Setup QEMU for cross-compilation + if: ${{ inputs.platform == 'arm64' }} + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ inputs.platform }} + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: ${{ inputs.platform }} + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: | + root-ccache + key: ccache-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/*.cpp') }} + restore-keys: | + ccache-${{ inputs.platform }}-${{ inputs.name }}- + ccache-${{ inputs.platform }}- + + - name: Restore apt-get + uses: actions/cache/restore@v4 + with: + path: | + var-cache-apt + key: apt-get-${{ inputs.platform }}-${{ inputs.name }}-${{ hashFiles('src/**/package.xml') }} + restore-keys: | + apt-get-${{ inputs.platform }}-${{ inputs.name }}- + apt-get-${{ inputs.platform }}- + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "root-ccache": "/root/.ccache", + "var-cache-apt": "/var/cache/apt" + } + skip-extraction: true + + - name: Set Docker tags + id: set-docker-tags + run: | + tags=() + if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then + tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") + fi + + tags+=("{{date 'YYYYMMDD'}}") + tags+=("latest") + tags+=("latest-${{ inputs.tag-prefix }}") + + # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "tags<<$EOF" >> $GITHUB_OUTPUT + echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + shell: bash + + - name: Docker meta for autoware:base + id: meta-base + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-base + flavor: | + latest=false + suffix=-base${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:core-devel + id: meta-core-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-core-devel + flavor: | + latest=false + suffix=-core-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception-devel + id: meta-universe-sensing-perception-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception-devel + flavor: | + latest=false + suffix=-universe-sensing-perception-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-sensing-perception + id: meta-universe-sensing-perception + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-sensing-perception + flavor: | + latest=false + suffix=-universe-sensing-perception${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping-devel + id: meta-universe-localization-mapping-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping-devel + flavor: | + latest=false + suffix=-universe-localization-mapping-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-localization-mapping + id: meta-universe-localization-mapping + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-localization-mapping + flavor: | + latest=false + suffix=-universe-localization-mapping${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe-devel + id: meta-universe-devel + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe-devel + flavor: | + latest=false + suffix=-universe-devel${{ inputs.tag-suffix }} + + - name: Docker meta for autoware:universe + id: meta-universe + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-universe + flavor: | + latest=auto + suffix=-universe${{ inputs.tag-suffix }} + + - name: Build and Push to GitHub Container Registry + uses: docker/bake-action@v5 + with: + push: false + files: | + docker/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} + ${{ steps.meta-core-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception-devel.outputs.bake-file }} + ${{ steps.meta-universe-sensing-perception.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping-devel.outputs.bake-file }} + ${{ steps.meta-universe-localization-mapping.outputs.bake-file }} + ${{ steps.meta-universe-devel.outputs.bake-file }} + ${{ steps.meta-universe.outputs.bake-file }} + provenance: false + set: | + ${{ inputs.build-args }} diff --git a/.github/workflows/docker-build-and-push-arm64.yaml b/.github/workflows/docker-build-and-push-arm64.yaml index 9a4dcc4695..11698b0de1 100644 --- a/.github/workflows/docker-build-and-push-arm64.yaml +++ b/.github/workflows/docker-build-and-push-arm64.yaml @@ -1,88 +1,88 @@ -name: docker-build-and-push-arm64 - -on: - push: - branches: - - main - tags: - workflow_dispatch: - -jobs: - load-env: - uses: ./.github/workflows/load-env.yaml - - docker-build-and-push: - needs: load-env - runs-on: [self-hosted, linux, ARM64] - strategy: - fail-fast: false - matrix: - name: - - no-cuda - - cuda - include: - - name: no-cuda - platform: arm64 - base_image_env: base_image - lib_dir: aarch64 - setup-args: --no-nvidia - additional-tag-suffix: "" - - name: cuda - platform: arm64 - base_image_env: base_image - lib_dir: aarch64 - additional-tag-suffix: -cuda - steps: - # https://github.com/actions/checkout/issues/211 - - name: Change permission of workspace - run: | - sudo chown -R $USER:$USER ${{ github.workspace }} - - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set git config - uses: autowarefoundation/autoware-github-actions/set-git-config@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Free disk space - uses: ./.github/actions/free-disk-space - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - *.env - *.repos - .github/actions/docker-build-and-push/action.yaml - .github/workflows/docker-build-and-push*.yaml - ansible-galaxy-requirements.yaml - ansible/** - docker/** - - - name: Build 'Autoware' - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref_type == 'tag') }} - uses: ./.github/actions/docker-build-and-push - with: - name: ${{ matrix.name }} - platform: ${{ matrix.platform }} - bake-target: autoware - build-args: | - *.platform=linux/${{ matrix.platform }} - *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} - *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} - *.args.SETUP_ARGS=${{ matrix.setup-args }} - *.args.LIB_DIR=${{ matrix.lib_dir }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main - *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max - tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} - tag-prefix: ${{ needs.load-env.outputs.rosdistro }} - - - name: Show disk space - run: | - df -h +name: docker-build-and-push-arm64 + +on: + push: + branches: + - main + tags: + workflow_dispatch: + +jobs: + load-env: + uses: ./.github/workflows/load-env.yaml + + docker-build-and-push: + needs: load-env + runs-on: [self-hosted, linux, ARM64] + strategy: + fail-fast: false + matrix: + name: + - no-cuda + - cuda + include: + - name: no-cuda + platform: arm64 + base_image_env: base_image + lib_dir: aarch64 + setup-args: --no-nvidia + additional-tag-suffix: "" + - name: cuda + platform: arm64 + base_image_env: base_image + lib_dir: aarch64 + additional-tag-suffix: -cuda + steps: + # https://github.com/actions/checkout/issues/211 + - name: Change permission of workspace + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} + + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + *.env + *.repos + .github/actions/docker-build-and-push/action.yaml + .github/workflows/docker-build-and-push*.yaml + ansible-galaxy-requirements.yaml + ansible/** + docker/** + + - name: Build 'Autoware' + if: ${{ steps.changed-files.outputs.any_changed == 'true' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && github.ref_type == 'tag') }} + uses: ./.github/actions/docker-build-and-push + with: + name: ${{ matrix.name }} + platform: ${{ matrix.platform }} + bake-target: autoware + build-args: | + *.platform=linux/${{ matrix.platform }} + *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} + *.args.SETUP_ARGS=${{ matrix.setup-args }} + *.args.LIB_DIR=${{ matrix.lib_dir }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main + *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max + tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} + tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + + - name: Show disk space + run: | + df -h diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml index 9199634b8c..c29bb57983 100644 --- a/.github/workflows/docker-build-and-push.yaml +++ b/.github/workflows/docker-build-and-push.yaml @@ -1,83 +1,126 @@ -name: docker-build-and-push - -on: - push: - branches: - - main - tags: - workflow_dispatch: - -jobs: - load-env: - uses: ./.github/workflows/load-env.yaml - - docker-build-and-push: - needs: load-env - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - name: - - no-cuda - - cuda - include: - - name: no-cuda - platform: amd64 - base_image_env: base_image - lib_dir: x86_64 - setup-args: --no-nvidia - additional-tag-suffix: "" - - name: cuda - platform: amd64 - base_image_env: base_image - lib_dir: x86_64 - additional-tag-suffix: -cuda - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set git config - uses: autowarefoundation/autoware-github-actions/set-git-config@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Free disk space - uses: ./.github/actions/free-disk-space - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - *.env - *.repos - .github/actions/docker-build-and-push/action.yaml - .github/workflows/docker-build-and-push*.yaml - ansible-galaxy-requirements.yaml - ansible/** - docker/** - - - name: Build 'Autoware' - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref_type == 'tag') }} - uses: ./.github/actions/docker-build-and-push - with: - name: ${{ matrix.name }} - platform: ${{ matrix.platform }} - bake-target: autoware - build-args: | - *.platform=linux/${{ matrix.platform }} - *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} - *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} - *.args.SETUP_ARGS=${{ matrix.setup-args }} - *.args.LIB_DIR=${{ matrix.lib_dir }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main - *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max - tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} - tag-prefix: ${{ needs.load-env.outputs.rosdistro }} - - - name: Show disk space - run: | - df -h +name: docker-build-and-push + +on: + push: + branches: + - main + tags: + workflow_dispatch: + +jobs: + load-env: + uses: ./.github/workflows/load-env.yaml + + docker-build: + needs: load-env + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: + - no-cuda + - cuda + include: + - name: amd64-no-cuda + platform: amd64 + base_image_env: base_image + lib_dir: x86_64 + setup-args: --no-nvidia + additional-tag-suffix: "" + - name: amd64-cuda + platform: amd64 + base_image_env: base_image + lib_dir: x86_64 + additional-tag-suffix: -cuda + - name: arm64-no-cuda + platform: arm64 + base_image_env: base_image + lib_dir: aarch64 + setup-args: --no-nvidia + additional-tag-suffix: "" + - name: arm64-cuda + platform: arm64 + base_image_env: base_image + lib_dir: aarch64 + additional-tag-suffix: -cuda + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + *.env + *.repos + .github/actions/docker-build-and-push/action.yaml + .github/workflows/docker-build-and-push*.yaml + ansible-galaxy-requirements.yaml + ansible/** + docker/** + + - name: Build 'Autoware' + if: ${{ steps.changed-files.outputs.any_changed == 'true' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && github.ref_type == 'tag') }} + uses: ./.github/actions/docker-buildv2 + with: + name: ${{ matrix.name }} + platform: ${{ matrix.platform }} + bake-target: autoware + build-args: | + *.platform=linux/${{ matrix.platform }} + *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} + *.args.SETUP_ARGS=${{ matrix.setup-args }} + *.args.LIB_DIR=${{ matrix.lib_dir }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-main + *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:${{ matrix.name }}-${{ matrix.platform }}-${{ github.ref_name }},mode=max + tag-suffix: ${{ matrix.additional-tag-suffix }}-${{ matrix.platform }} + tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + + - name: Show disk space + run: | + df -h + + combine-and-push-multi-arch: + needs: docker-build + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Combine and push multi-arch images + run: | + for variant in "no-cuda" "cuda"; do + docker buildx imagetools create -t ghcr.io/${{ github.repository }}:${{ needs.load-env.outputs.rosdistro }}-${variant} \ + ghcr.io/${{ github.repository }}:${{ needs.load-env.outputs.rosdistro }}-${variant}-amd64 \ + ghcr.io/${{ github.repository }}:${{ needs.load-env.outputs.rosdistro }}-${variant}-arm64 + done + + # Push the combined images to the repository + for variant in "no-cuda" "cuda"; do + docker push ghcr.io/${{ github.repository }}:${{ needs.load-env.outputs.rosdistro }}-${variant} + done + + - name: Show pushed images + run: | + docker buildx imagetools inspect ghcr.io/${{ github.repository }}:${{ needs.load-env.outputs.rosdistro }}-no-cuda + docker buildx imagetools inspect ghcr.io/${{ github.repository }}:${{ needs.load-env.outputs.rosdistro }}-cuda