diff --git a/.github/workflows/_get-image-list.yml b/.github/workflows/_get-image-list.yml new file mode 100644 index 000000000..f04925cf7 --- /dev/null +++ b/.github/workflows/_get-image-list.yml @@ -0,0 +1,54 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Get Image List +permissions: read-all +on: + workflow_call: + inputs: + examples: + default: "" + required: false + type: string + images: + default: "" + required: false + type: string + outputs: + matrix: + description: "Image List" + value: ${{ jobs.get-image-list.outputs.matrix }} + +jobs: + get-image-list: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.get-matrix.outputs.matrix }} + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Set Matrix + id: get-matrix + run: | + image_list=[] + if [[ ! -z "${{ inputs.examples }}" ]]; then + pip install yq + examples=($(echo ${{ inputs.examples }} | tr ',' ' ')) + for example in ${examples[@]} + do + images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.') + image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) + done + fi + + if [[ ! -z "${{ inputs.images }}" ]]; then + images=($(echo ${{ inputs.images }} | tr ',' ' ')) + input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.') + image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list})) + fi + + echo "print image list..." + echo "$image_list" | jq . | jq -r '.[]' + echo "end of image list..." + echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/manual-docker-publish.yml b/.github/workflows/manual-docker-publish.yml index 42e1da3c4..8b20eee27 100644 --- a/.github/workflows/manual-docker-publish.yml +++ b/.github/workflows/manual-docker-publish.yml @@ -12,14 +12,14 @@ on: type: string examples: default: "Translation" - description: 'List of examples to test [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]' - required: true + description: 'List of examples to publish [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]' + required: false type: string - gmc: - default: false - description: 'Publish gmc images' + images: + default: "gmcmanager,gmcrouter" + description: 'List of images to publish [gmcmanager,gmcrouter, ...]' required: false - type: boolean + type: string tag: default: "v0.9" description: "Tag to publish" @@ -34,31 +34,10 @@ on: permissions: read-all jobs: get-image-list: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.scan-matrix.outputs.matrix }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 - - - name: Set Matrix - id: scan-matrix - run: | - pip install yq - examples=($(echo ${{ inputs.examples }} | tr ',' ' ')) - image_list=[] - for example in ${examples[@]} - do - echo ${example} - images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.') - image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) - done - - if [ "${{ inputs.gmc }}" == "true" ]; then - image_list=$(echo ${image_list} | jq -c '. + ["gmcmanager","gmcrouter"]') - fi - echo $image_list - echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT + uses: ./.github/workflows/_get-image-list.yml + with: + examples: ${{ inputs.examples }} + images: ${{ inputs.images }} publish: needs: [get-image-list] diff --git a/.github/workflows/manual-docker-scan.yml b/.github/workflows/manual-docker-scan.yml index 7b2697803..afb73a168 100644 --- a/.github/workflows/manual-docker-scan.yml +++ b/.github/workflows/manual-docker-scan.yml @@ -13,11 +13,11 @@ on: examples: default: "ChatQnA" description: 'List of examples to scan [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]' - required: true + required: false type: string images: - default: "" - description: 'List of images to scan' + default: "gmcmanager,gmcrouter" + description: 'List of images to scan [gmcmanager,gmcrouter, ...]' required: false type: string tag: @@ -39,44 +39,24 @@ on: permissions: read-all jobs: get-image-list: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.scan-matrix.outputs.matrix }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 + uses: ./.github/workflows/_get-image-list.yml + with: + examples: ${{ inputs.examples }} + images: ${{ inputs.images }} - - name: Set Matrix - id: scan-matrix - run: | - if [[ ! -z "${{ inputs.images }}" ]]; then - images=($(echo ${{ inputs.images }} | tr ',' ' ')) - image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.') - else - pip install yq - examples=($(echo ${{ inputs.examples }} | tr ',' ' ')) - image_list=[] - for example in ${examples[@]} - do - images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.') - image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) - done - fi - - echo "print image list..." - echo "$image_list" | jq . | jq -r '.[]' - echo "end of image list..." - echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT - - scan-license: + scan-docker: needs: get-image-list - if: ${{ fromJSON(inputs.sbom_scan) }} || ${{ fromJSON(inputs.trivy_scan) }} runs-on: "docker-build-${{ inputs.node }}" strategy: matrix: image: ${{ fromJson(needs.get-image-list.outputs.matrix) }} fail-fast: false steps: + - name: Clean up Working Directory + run: | + sudo rm -rf ${{github.workspace}}/* || true + docker system prune -f + - name: Pull Image run: | docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} @@ -84,7 +64,7 @@ jobs: - name: SBOM Scan Container uses: anchore/sbom-action@v0.17.1 - if: always() && ${{ inputs.sbom_scan }} + if: ${{ inputs.sbom_scan }} with: image: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }} output-file: ${{ matrix.image }}-sbom-scan.txt @@ -92,7 +72,7 @@ jobs: - name: Security Scan Container uses: aquasecurity/trivy-action@0.24.0 - if: always() && ${{ inputs.trivy_scan }} + if: ${{ inputs.trivy_scan }} with: image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }} output: ${{ matrix.image }}-trivy-scan.txt @@ -104,18 +84,30 @@ jobs: - name: Cleanup if: always() - run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} + run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} || true + + - name: Collect Logs + if: always() + run: | + mkdir -p /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} + mv ${{ matrix.image }}-*-scan.txt /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} + upload-artifacts: + needs: scan-docker + runs-on: "docker-build-${{ inputs.node }}" + if: always() + steps: - uses: actions/upload-artifact@v4.3.4 - if: always() && (${{ inputs.sbom_scan }} with: - name: sbom-scan - path: ${{ matrix.image }}-*-sbom-scan.txt + name: sbom-scan-${{ inputs.tag }}-${{ github.run_number }} + path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-sbom-scan.txt overwrite: true - uses: actions/upload-artifact@v4.3.4 - if: always() && (${{ inputs.trivy_scan }} with: - name: trivy-scan - path: ${{ matrix.image }}-*-trivy-scan.txt + name: trivy-scan-${{ inputs.tag }}-${{ github.run_number }} + path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-trivy-scan.txt overwrite: true + + - name: Remove Logs + run: rm -rf /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} && rm -rf /tmp/sbom-action-*