staging-release.yaml

---
name: Release from staging

# This is only expected to be invoked on-demand by a specific user.
on:
  workflow_dispatch:
    inputs:
      version:
        type: string
        description: The version we want to release from staging, ensure this is numeric without the v prefix for the tag.
        required: true
      docker-image:
        type: string
        description: Optionally override the image name to push to on Docker Hub.
        default: fluent/fluent-bit
        required: false
      github-image:
        type: string
        description: Optionally override the image name to push to on Github Container Registry.
        default: fluent/fluent-bit
        required: false

# We do not want a new staging build to run whilst we are releasing the current staging build.
# We also do not want multiples to run for the same version.
concurrency: staging-build-release

env:
  STAGING_IMAGE_NAME: ghcr.io/${{ github.repository }}/staging

jobs:

  staging-release-version-check:
    name: Check staging release matches
    environment: release # required to get bucket name
    runs-on: ubuntu-latest
    outputs:
      major-version: ${{ steps.get_major_version.outputs.value }}
    permissions:
      contents: read
    steps:
    - name: Get the version on staging
      run: |
        curl --fail -LO "$AWS_URL/latest-version.txt"
        cat latest-version.txt
        STAGING_VERSION=$(cat latest-version.txt)
        [[ "$STAGING_VERSION" != "$RELEASE_VERSION" ]] && echo "Latest version mismatch: $STAGING_VERSION != $RELEASE_VERSION" && exit 1
        # Must end in something that exits 0
        echo "Successfully confirmed version is as expected: $STAGING_VERSION"
      shell: bash
      env:
        AWS_URL: https://${{ secrets.AWS_S3_BUCKET_STAGING }}.s3.amazonaws.com
        RELEASE_VERSION: ${{ github.event.inputs.version }}

    # Get the major version, i.e. 1.9.3 --> 1.9, or just return the passed in version.
    - name: Convert to major version format
      id: get_major_version
      run: |
        MAJOR_VERSION="$RELEASE_VERSION"
        if [[ $RELEASE_VERSION =~ ^[0-9]+\.[0-9]+ ]]; then
          MAJOR_VERSION="${BASH_REMATCH[0]}"
        fi
        echo "value=$MAJOR_VERSION" >> $GITHUB_OUTPUT
      shell: bash
      env:
        RELEASE_VERSION: ${{ github.event.inputs.version }}

    - name: Checkout repository
      uses: actions/checkout@v4

    # Check we can download the AppVeyor build which confirms it matches the version to release as well as being a successful build
    - name: Get Appveyor binaries
      run: |
        ./packaging/appveyor-download.sh
      shell: bash
      env:
        TAG: v${{ github.event.inputs.version }}

  staging-release-generate-package-matrix:
    name: Get package matrix
    runs-on: ubuntu-latest
    outputs:
      deb-build-matrix: ${{ steps.get-matrix.outputs.deb-build-matrix }}
      rpm-build-matrix: ${{ steps.get-matrix.outputs.rpm-build-matrix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup runner
        run: |
          sudo apt-get update
          sudo apt-get install -y jq
        shell: bash

      # Cope with 1.9 as well as 2.0
      - uses: ./.github/actions/generate-package-build-matrix
        id: get-matrix
        with:
          ref: v${{ inputs.version }}

      # Now annotate with whether it is Yum or Apt based

  # 1. Take packages from the staging bucket
  # 2. Sign them with the release GPG key
  # 3. Also take existing release packages from the release bucket.
  # 4. Create a full repo configuration using the existing releases as well.
  # 5. Upload to release bucket.
  # Note we could resign all packages as well potentially if we wanted to update the key.
  staging-release-yum-packages:
    name: S3 - update YUM packages bucket
    runs-on: ubuntu-22.04 # no createrepo on Ubuntu 20.04
    environment: release
    needs:
      - staging-release-version-check
      - staging-release-generate-package-matrix
    permissions:
      contents: read
    strategy:
      matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.rpm-build-matrix) }}
      fail-fast: false
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Setup runner
      run: |
        sudo apt-get update
        sudo apt-get install -y createrepo-c rpm
      shell: bash

    - name: Import GPG key for signing
      id: import_gpg
      uses: crazy-max/ghaction-import-gpg@v6
      with:
        gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
        passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}

    # Download the current release bucket
    # Add everything from staging
    # Sign and set up metadata for it all
    # Upload to release bucket

    - name: Sync packages from buckets on S3
      run: |
        mkdir -p "packaging/releases/$DISTRO"
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        DISTRO: ${{ matrix.distro }}
      shell: bash

    - name: GPG set up keys for signing
      run: |
        gpg --export -a "${{ steps.import_gpg.outputs.name }}" > /tmp/fluentbit.key
        rpm --import /tmp/fluentbit.key
      shell: bash

    - name: Update repo info and remove any staging details
      run: |
        packaging/update-yum-repo.sh
      env:
        GPG_KEY: ${{ steps.import_gpg.outputs.name }}
        AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }}
        VERSION: ${{ github.event.inputs.version }}
        BASE_PATH: "packaging/releases"
        RPM_REPO: ${{ matrix.distro }}
      shell: bash

    - name: Sync to release bucket on S3
      run: |
        aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        DISTRO: ${{ matrix.distro }}
      shell: bash

  staging-release-apt-packages:
    name: S3 - update APT packages bucket
    runs-on: ubuntu-latest
    environment: release
    needs:
      - staging-release-version-check
      - staging-release-generate-package-matrix
    permissions:
      contents: read
    strategy:
      matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.deb-build-matrix) }}
      fail-fast: false
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Setup runner
      run: |
        sudo apt-get update
        sudo apt-get install -y aptly debsigs distro-info rsync
      shell: bash

    - name: Convert version to codename
      id: get_codename
      run: |
        CODENAME="$DISTRO"
        if [[ "$DISTRO" == ubuntu* ]]; then
          echo "Converting Ubuntu version to codename"
          UBUNTU_NAME=$(grep "${DISTRO##*/} LTS" /usr/share/distro-info/ubuntu.csv|cut -d ',' -f3)
          echo "Got Ubuntu codename: $UBUNTU_NAME"
          CODENAME="ubuntu/$UBUNTU_NAME"
        fi
        echo "Using codename: $CODENAME"
        echo "CODENAME=$CODENAME" >> $GITHUB_OUTPUT
      shell: bash
      env:
        DISTRO: ${{ matrix.distro }}

    - name: Import GPG key for signing
      id: import_gpg
      uses: crazy-max/ghaction-import-gpg@v6
      with:
        gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
        passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}

    - name: Sync packages from buckets on S3
      run: |
        mkdir -p "packaging/releases/$CODENAME"
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        CODENAME: ${{ steps.get_codename.outputs.CODENAME }}
      shell: bash

    - name: Update repo info and remove any staging details
      run: |
        packaging/update-apt-repo.sh
      env:
        GPG_KEY: ${{ steps.import_gpg.outputs.name }}
        AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }}
        VERSION: ${{ github.event.inputs.version }}
        BASE_PATH: "packaging/releases"
        DEB_REPO: ${{ steps.get_codename.outputs.CODENAME }}
      shell: bash

    - name: Sync to release bucket on S3
      run: |
        aws s3 sync "packaging/releases/$CODENAME" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" --delete --follow-symlinks --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        CODENAME: ${{ steps.get_codename.outputs.CODENAME }}
      shell: bash

  staging-release-update-non-linux-s3:
    name: Update Windows and macOS packages
    runs-on: ubuntu-22.04
    environment: release
    needs:
      - staging-release-version-check
    permissions:
      contents: none
    strategy:
      matrix:
        distro:
          - macos
          - windows
      fail-fast: false
    steps:
    - name: Sync packages from buckets on S3
      run: |
        mkdir -p "packaging/releases/$DISTRO"
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        DISTRO: ${{ matrix.distro }}
      shell: bash

    - name: Sync to release bucket on S3
      run: |
        aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        DISTRO: ${{ matrix.distro }}
      shell: bash

  staging-release-update-base-s3:
    name: Update top-level bucket info
    runs-on: ubuntu-22.04
    environment: release
    needs:
      - staging-release-apt-packages
      - staging-release-yum-packages
    permissions:
      contents: none
    steps:
    - name: Import GPG key for signing
      id: import_gpg
      uses: crazy-max/ghaction-import-gpg@v6
      with:
        gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
        passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}

    - name: GPG public key
      run: |
        gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./fluentbit.key
        aws s3 cp ./fluentbit.key s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/fluentbit.key --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
      shell: bash

    - name: JSON schema
      continue-on-error: true
      run: |
        aws s3 sync "s3://${AWS_STAGING_S3_BUCKET}/${VERSION}" "s3://${AWS_RELEASE_S3_BUCKET}/${VERSION}" --no-progress
      env:
          VERSION: ${{ github.event.inputs.version }}
          AWS_REGION: "us-east-1"
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_STAGING_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }}
          AWS_RELEASE_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }}
      shell: bash

  staging-release-source-s3:
    name: S3 - update source bucket
    runs-on: ubuntu-latest
    environment: release
    needs:
      - staging-release-version-check
    permissions:
      contents: read
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Sync packages from buckets on S3
      run: |
        mkdir -p release staging
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" release/ --no-progress
        aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/source/" staging/ --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
      shell: bash

    - name: Get Appveyor binaries
      run: |
        ./packaging/appveyor-download.sh
      shell: bash
      env:
        TAG: v${{ github.event.inputs.version }}
        OUTPUT_DIR: appveyor

    - name: Move components from staging and setup
      run: |
        ./packaging/update-source-packages.sh
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
        SOURCE_DIR: staging
        WINDOWS_SOURCE_DIR: appveyor
        TARGET_DIR: release
        VERSION: ${{ github.event.inputs.version }}
        MAJOR_VERSION: ${{ needs.staging-release-version-check.outputs.major-version }}
      shell: bash

    - name: Sync to bucket on S3
      run: |
        aws s3 sync release/ "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" --delete --follow-symlinks --no-progress
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_REGION: "us-east-1"
      shell: bash

  # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing.
  # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533
  staging-release-images:
    name: Release ${{ matrix.tag }} Linux container images
    runs-on: ubuntu-latest
    needs:
      - staging-release-version-check
    environment: release
    strategy:
      fail-fast: false
      matrix:
        # All the explicit tags we want to release
        tag: [
          "${{ github.event.inputs.version }}",
          "${{ needs.staging-release-version-check.outputs.major-version }}",
          "${{ github.event.inputs.version }}-debug",
          "${{ needs.staging-release-version-check.outputs.major-version }}-debug",
        ]
    permissions:
      packages: write
    steps:
      # Primarily because the skopeo errors are hard to parse and non-obvious
      - name: Check the image exists
        run: |
          docker pull "$STAGING_IMAGE_NAME:$TAG"
        env:
          TAG: ${{ matrix.tag }}
        shell: bash

      # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it.
      - name: Promote container images from staging to Dockerhub
        run: |
          docker run --rm  \
            quay.io/skopeo/stable:latest \
            copy \
              --all \
              --retry-times 10 \
              --src-no-creds \
              --dest-creds "$RELEASE_CREDS" \
              "docker://$STAGING_IMAGE_NAME:$TAG" \
              "docker://$RELEASE_IMAGE_NAME:$TAG"
        env:
          RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
          RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}
          TAG: ${{ matrix.tag }}
        shell: bash

      - name: Promote container images from staging to GHCR.io
        if: ${{ startsWith(github.event.inputs.version, '2.0') || startsWith(github.event.inputs.version, '2.1') || ! startsWith(matrix.tag, 'latest') }}
        run: |
          docker run --rm  \
            quay.io/skopeo/stable:latest \
            copy \
              --all \
              --retry-times 10 \
              --src-no-creds \
              --dest-creds "$RELEASE_CREDS" \
              "docker://$STAGING_IMAGE_NAME:$TAG" \
              "docker://$RELEASE_IMAGE_NAME:$TAG"
        env:
          RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
          RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}
          TAG: ${{ matrix.tag }}
        shell: bash

  # Part of resolution for: https://github.com/fluent/fluent-bit/issues/7748
  # More recent build-push-actions may mean legacy format is not preserved so we provide arch-specific tags just in case
  staging-release-images-arch-specific-legacy-tags:
    name: Release ${{ matrix.arch }} legacy format Linux container images
    runs-on: ubuntu-latest
    needs:
      - staging-release-images
    environment: release
    strategy:
      fail-fast: false
      matrix:
        arch:
          - amd64
          - arm64
          - arm/v7
    permissions:
      packages: write
    env:
      RELEASE_IMAGE_NAME: ${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
      RELEASE_TAG: ${{ github.event.inputs.version }}
    steps:

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Convert arch to tag
        id: get-tag
        run: |
          TAG="${RELEASE_TAG}-${{ matrix.arch }}"
          echo "Input value: $TAG"
          TAG=${TAG/\//-}
          echo "Using tag: $TAG"
          echo "tag=$TAG" >> $GITHUB_OUTPUT
        shell: bash

      - name: Pull release image
        run: docker pull --platform='linux/${{ matrix.arch }}' "$RELEASE_IMAGE_NAME:$RELEASE_TAG"
        shell: bash

      - name: Tag and push legacy format image to DockerHub
        run: |
          docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" docker.io/"$RELEASE_IMAGE_NAME:$TAG"
          docker push docker.io/"$RELEASE_IMAGE_NAME:$TAG"
        shell: bash
        env:
          TAG: ${{ steps.get-tag.outputs.tag }}

      - name: Tag and push legacy format image to Github Container Registry
        run: |
          docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" ghcr.io/"$RELEASE_IMAGE_NAME:$TAG"
          docker push ghcr.io/"$RELEASE_IMAGE_NAME:$TAG"
        shell: bash
        env:
          TAG: ${{ steps.get-tag.outputs.tag }}

  staging-release-images-latest-tags:
    # Only update latest tags for 2.2 releases
    if: startsWith(github.event.inputs.version, '2.2')
    name: Release latest Linux container images
    runs-on: ubuntu-latest
    needs:
      - staging-release-images
    environment: release
    strategy:
      fail-fast: false
      matrix:
        tag: [
          "latest",
          "latest-debug"
        ]
    permissions:
      packages: write
    steps:
      # Primarily because the skopeo errors are hard to parse and non-obvious
      - name: Check the image exists
        run: |
          docker pull "$STAGING_IMAGE_NAME:$TAG"
        env:
          TAG: ${{ matrix.tag }}
        shell: bash

      # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it.
      - name: Promote container images from staging to Dockerhub
        run: |
          docker run --rm  \
            quay.io/skopeo/stable:latest \
            copy \
              --all \
              --retry-times 10 \
              --src-no-creds \
              --dest-creds "$RELEASE_CREDS" \
              "docker://$STAGING_IMAGE_NAME:$TAG" \
              "docker://$RELEASE_IMAGE_NAME:$TAG"
        env:
          RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
          RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}
          TAG: ${{ matrix.tag }}
        shell: bash

      - name: Promote container images from staging to GHCR.io
        run: |
          docker run --rm  \
            quay.io/skopeo/stable:latest \
            copy \
              --all \
              --retry-times 10 \
              --src-no-creds \
              --dest-creds "$RELEASE_CREDS" \
              "docker://$STAGING_IMAGE_NAME:$TAG" \
              "docker://$RELEASE_IMAGE_NAME:$TAG"
        env:
          RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
          RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}
          TAG: ${{ matrix.tag }}
        shell: bash

  staging-release-images-windows:
    name: Release Windows images
    # Cannot be done by Skopeo on a Linux runner unfortunately
    runs-on: windows-latest
    needs:
      - staging-release-version-check
    environment: release
    permissions:
      packages: write
    strategy:
      fail-fast: false
      matrix:
        tag: [
          "windows-2019-${{ github.event.inputs.version }}",
          "windows-2022-${{ github.event.inputs.version }}"
        ]
    steps:
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Check the image exists
        run: |
          docker pull "$STAGING_IMAGE_NAME:$TAG"
        env:
          TAG: ${{ matrix.tag }}
        shell: bash

      - name: Promote container images from staging to GHCR.io
        run: |
          docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG"
          docker push "$RELEASE_IMAGE_NAME:$TAG"
        env:
          RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
          RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}
          TAG: ${{ matrix.tag }}
        shell: bash

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Promote container images from staging to Dockerhub
        run: |
          docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG"
          docker push "$RELEASE_IMAGE_NAME:$TAG"
        env:
          RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
          RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}
          TAG: ${{ matrix.tag }}
        shell: bash

  staging-release-images-sign:
    name: Sign container image manifests
    permissions: write-all
    runs-on: ubuntu-latest
    environment: release
    needs:
      - staging-release-images
    env:
      DH_RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
      GHCR_RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
    steps:
      - name: Install cosign
        uses: sigstore/cosign-installer@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Cosign with a key
        # Only run if we have a key defined
        if: ${{ env.COSIGN_PRIVATE_KEY }}
        # The key needs to cope with newlines
        run: |
          echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key
          cosign sign --key /tmp/my_cosign.key --recursive \
            -a "repo=${{ github.repository }}" \
            -a "workflow=${{ github.workflow }}" \
            -a "release=${{ github.event.inputs.version }}" \
            "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
            "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \
            "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
            "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug"
          rm -f /tmp/my_cosign.key
        shell: bash
        env:
          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
          COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional

      - name: Cosign keyless signing using Rektor public transparency log
        # This step uses the identity token to provision an ephemeral certificate
        # against the sigstore community Fulcio instance, and records it to the
        # sigstore community Rekor transparency log.
        #
        # We use recursive signing on the manifest to cover all the images.
        run: |
          cosign sign --yes --recursive \
            -a "repo=${{ github.repository }}" \
            -a "workflow=${{ github.workflow }}" \
            -a "release=${{ github.event.inputs.version }}" \
            "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
            "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \
            "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
            "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug"
        shell: bash
        env:
          COSIGN_EXPERIMENTAL: true

  staging-release-upload-cosign-key:
    name: Upload Cosign public key for verification
    needs:
      - staging-release-images-sign
    permissions:
      contents: none
    runs-on: ubuntu-latest
    steps:
      - name: Install cosign
        uses: sigstore/cosign-installer@v2

      - name: Get public key and add to S3 bucket
        # Only run if we have a key defined
        if: ${{ env.COSIGN_PRIVATE_KEY }}
        # The key needs to cope with newlines
        run: |
          echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key
          cosign public-key --key /tmp/my_cosign.key > ./cosign.pub
          rm -f /tmp/my_cosign.key
          cat ./cosign.pub
          aws s3 cp ./cosign.pub "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/cosign.pub" --no-progress
        shell: bash
        env:
          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
          COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: "us-east-1"

  staging-release-smoke-test-packages:
    name: Run package smoke tests
    permissions:
      contents: read
    runs-on: ubuntu-latest
    environment: release
    needs:
      - staging-release-apt-packages
      - staging-release-yum-packages
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Test release packages
        run: |
          ./packaging/test-release-packages.sh
        shell: bash
        env:
          VERSION_TO_CHECK_FOR: ${{ github.event.inputs.version }}
          FLUENT_BIT_PACKAGES_URL: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com
          FLUENT_BIT_PACKAGES_KEY: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com/fluentbit.key

  staging-release-smoke-test-containers:
    name: Run container smoke tests
    permissions:
      contents: read
      packages: read
    runs-on: ubuntu-latest
    environment: release
    needs:
      - staging-release-images
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Test containers
        run: |
          ./packaging/testing/smoke/container/container-smoke-test.sh
        shell: bash
        env:
          IMAGE_TAG: ${{ github.event.inputs.version }}

  staging-release-create-release:
    name: Create the Github Release once packages and containers are up
    needs:
      - staging-release-images
      - staging-release-apt-packages
      - staging-release-yum-packages
    permissions:
      contents: write
    environment: release
    runs-on: ubuntu-latest
    steps:
      - name: Release 2.0 - not latest
        uses: softprops/action-gh-release@v1
        if: startsWith(inputs.version, '2.0')
        with:
          body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
          draft: false
          generate_release_notes: true
          name: "Fluent Bit ${{ inputs.version }}"
          tag_name: v${{ inputs.version }}
          target_commitish: '2.0'

      - name: Release 2.1 - not latest
        uses: softprops/action-gh-release@v1
        if: startsWith(inputs.version, '2.1')
        with:
          body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
          draft: false
          generate_release_notes: true
          name: "Fluent Bit ${{ inputs.version }}"
          tag_name: v${{ inputs.version }}
          target_commitish: '2.1'

      - name: Release 2.2 and latest
        uses: softprops/action-gh-release@v1
        if: startsWith(inputs.version, '2.2')
        with:
          body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
          draft: false
          generate_release_notes: true
          name: "Fluent Bit ${{ inputs.version }}"
          tag_name: v${{ inputs.version }}

  staging-release-windows-checksums:
    name: Get Windows checksums for new release
    runs-on: ubuntu-22.04
    environment: release
    needs:
      - staging-release-update-non-linux-s3
    permissions:
      contents: none
    outputs:
      windows-exe32-hash: ${{ steps.hashes.outputs.WIN_32_EXE_HASH }}
      windows-zip32-hash: ${{ steps.hashes.outputs.WIN_32_ZIP_HASH }}
      windows-exe64-hash: ${{ steps.hashes.outputs.WIN_64_EXE_HASH }}
      windows-zip64-hash: ${{ steps.hashes.outputs.WIN_64_ZIP_HASH }}
      windows-arm-exe64-hash: ${{ steps.hashes.outputs.WIN_64_ARM_EXE_HASH }}
      windows-arm-zip64-hash: ${{ steps.hashes.outputs.WIN_64_ARM_ZIP_HASH }}
    steps:
      - name: Sync release Windows directory to get checksums
        run:
          aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/windows" ./ --exclude "*" --include "*.sha256"
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: "us-east-1"

      - name: Provide output for documentation PR
        id: hashes
        # do not fail the build for this
        continue-on-error: true
        run: |
          ls -l
          export WIN_32_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}')
          export WIN_32_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}')
          export WIN_64_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}')
          export WIN_64_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}')
          if [[ -f "./fluent-bit-${{ inputs.version }}-winarm64.exe.sha256" ]]; then
            export WIN_64_ARM_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-winarm64.exe.sha256"|awk '{print $1}')
          fi
          if [[ -f "./fluent-bit-${{ inputs.version }}-winarm64.zip.sha256" ]]; then
            export WIN_64_ARM_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-winarm64.zip.sha256"|awk '{print $1}')
          fi
          set | grep WIN_
          echo WIN_32_EXE_HASH="$WIN_32_EXE_HASH" >> $GITHUB_OUTPUT
          echo WIN_32_ZIP_HASH="$WIN_32_ZIP_HASH" >> $GITHUB_OUTPUT
          echo WIN_64_EXE_HASH="$WIN_64_EXE_HASH" >> $GITHUB_OUTPUT
          echo WIN_64_ZIP_HASH="$WIN_64_ZIP_HASH" >> $GITHUB_OUTPUT
          echo WIN_64_ARM_EXE_HASH="$WIN_64_ARM_EXE_HASH" >> $GITHUB_OUTPUT
          echo WIN_64_ARM_ZIP_HASH="$WIN_64_ARM_ZIP_HASH" >> $GITHUB_OUTPUT
        shell: bash

  staging-release-create-docs-pr:
    name: Create docs updates for new release
    needs:
      - staging-release-images
      - staging-release-windows-checksums
    permissions:
      contents: none
    environment: release
    runs-on: ubuntu-latest
    steps:
      - name: Release 2.0 - not latest
        if: startsWith(inputs.version, '2.0')
        uses: actions/checkout@v4
        with:
          repository: fluent/fluent-bit-docs
          ref: 2.0
          token: ${{ secrets.GH_PA_TOKEN }}

      - name: Release 2.1 - not latest
        if: startsWith(inputs.version, '2.1')
        uses: actions/checkout@v4
        with:
          repository: fluent/fluent-bit-docs
          ref: 2.1
          token: ${{ secrets.GH_PA_TOKEN }}

      - name: Release 2.2 and latest
        if: startsWith(inputs.version, '2.2')
        uses: actions/checkout@v4
        with:
          repository: fluent/fluent-bit-docs
          token: ${{ secrets.GH_PA_TOKEN }}

      - name: Ensure we have the script we need
        run: |
          if [[ ! -f update-release-version-docs.sh ]] ; then
            git checkout update-release-version-docs.sh -- master
          fi
        shell: bash

      - name: Update versions
        # Uses https://github.com/fluent/fluent-bit-docs/blob/master/update-release-version-docs.sh
        run: |
          ./update-release-version-docs.sh
        shell: bash
        env:
          NEW_VERSION: ${{ inputs.version }}
          WIN_32_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-exe32-hash }}
          WIN_32_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-zip32-hash }}
          WIN_64_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-exe64-hash }}
          WIN_64_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-zip64-hash }}
          WIN_64_ARM_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-arm-exe64-hash }}
          WIN_64_ARM_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-arm-zip64-hash }}

      - name: Raise docs PR
        id: cpr
        uses: peter-evans/create-pull-request@v5
        with:
          commit-message: 'release: update to v${{ inputs.version }}'
          signoff: true
          delete-branch: true
          title: 'release: update to v${{ inputs.version }}'
          # We need workflows permission so have to use the GH_PA_TOKEN
          token: ${{ secrets.GH_PA_TOKEN }}
          labels: ci,automerge
          body: |
            Update release ${{ inputs.version }} version.
            - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
            - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request
          draft: false

      - name: Check outputs
        if: ${{ steps.cpr.outputs.pull-request-number }}
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

  staging-release-create-version-update-pr:
    name: Create version update PR for new release
    needs:
      - staging-release-create-release
    permissions:
      contents: write
      pull-requests: write
    environment: release
    runs-on: ubuntu-latest
    steps:
      - name: Release 2.0
        if: startsWith(inputs.version, '2.0')
        uses: actions/checkout@v4
        with:
          ref: 2.0

      - name: Release 2.1
        if: startsWith(inputs.version, '2.1')
        uses: actions/checkout@v4
        with:
          ref: 2.1

      - name: Release 2.2 and latest
        if: startsWith(inputs.version, '2.2')
        uses: actions/checkout@v4

      # Get the new version to use
      - name: 'Get next minor version'
        id: semvers
        uses: "WyriHaximus/github-action-next-semvers@v1"
        with:
          version: ${{ inputs.version }}
          strict: true

      - run: ./update_version.sh
        shell: bash
        env:
          NEW_VERSION: ${{ steps.semvers.outputs.patch }}
          # Ensure we use the PR action to do the work
          DISABLE_COMMIT: 'yes'

      - name: Raise FB PR to update version
        id: cpr
        uses: peter-evans/create-pull-request@v5
        with:
          commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}'
          signoff: true
          delete-branch: true
          title: 'release: update to ${{ steps.semvers.outputs.patch }}'
          labels: ci,automerge
          body: |
            Update next release to ${{ steps.semvers.outputs.patch }} version.
            - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
            - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request
          draft: false

      - name: Check outputs
        if: ${{ steps.cpr.outputs.pull-request-number }}
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"