docker image cd workflow enhance (#702)

Signed-off-by: chensuyue <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/_get-image-list.yml

@@ -0,0 +1,54 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Get Image List
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      examples:
+        default: ""
+        required: false
+        type: string
+      images:
+        default: ""
+        required: false
+        type: string
+    outputs:
+      matrix:
+        description: "Image List"
+        value: ${{ jobs.get-image-list.outputs.matrix }}
+
+jobs:
+  get-image-list:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Set Matrix
+        id: get-matrix
+        run: |
+          image_list=[]
+          if [[ ! -z "${{ inputs.examples }}" ]]; then
+              pip install yq
+              examples=($(echo ${{ inputs.examples }} | tr ',' ' '))
+              for example in ${examples[@]}
+              do
+                  images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.')
+                  image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
+              done
+          fi
+
+          if [[ ! -z "${{ inputs.images }}" ]]; then
+              images=($(echo ${{ inputs.images }} | tr ',' ' '))
+              input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.')
+              image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list}))
+          fi
+
+          echo "print image list..."
+          echo "$image_list" | jq . | jq -r '.[]'
+          echo "end of image list..."
+          echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT

.github/workflows/manual-docker-publish.yml

@@ -12,14 +12,14 @@ on:
         type: string
       examples:
         default: "Translation"
-        description: 'List of examples to test [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]'
-        required: true
+        description: 'List of examples to publish [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]'
+        required: false
         type: string
-      gmc:
-        default: false
-        description: 'Publish gmc images'
+      images:
+        default: "gmcmanager,gmcrouter"
+        description: 'List of images to publish [gmcmanager,gmcrouter, ...]'
         required: false
-        type: boolean
+        type: string
       tag:
         default: "v0.9"
         description: "Tag to publish"
@@ -34,31 +34,10 @@ on:
 permissions: read-all
 jobs:
   get-image-list:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.scan-matrix.outputs.matrix }}
-    steps:
-      - name: Checkout out Repo
-        uses: actions/checkout@v4
-
-      - name: Set Matrix
-        id: scan-matrix
-        run: |
-          pip install yq
-          examples=($(echo ${{ inputs.examples }} | tr ',' ' '))
-          image_list=[]
-          for example in ${examples[@]}
-          do
-              echo ${example}
-              images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.')
-              image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
-          done
-
-          if [ "${{ inputs.gmc }}" == "true" ]; then
-              image_list=$(echo ${image_list} | jq -c '. + ["gmcmanager","gmcrouter"]')
-          fi
-          echo $image_list
-          echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
+    uses: ./.github/workflows/_get-image-list.yml
+    with:
+      examples: ${{ inputs.examples }}
+      images: ${{ inputs.images }}
 
   publish:
     needs: [get-image-list]

.github/workflows/manual-docker-scan.yml

@@ -13,11 +13,11 @@ on:
       examples:
         default: "ChatQnA"
         description: 'List of examples to scan [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]'
-        required: true
+        required: false
         type: string
       images:
-        default: ""
-        description: 'List of images to scan'
+        default: "gmcmanager,gmcrouter"
+        description: 'List of images to scan [gmcmanager,gmcrouter, ...]'
         required: false
         type: string
       tag:
@@ -39,60 +39,40 @@ on:
 permissions: read-all
 jobs:
   get-image-list:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.scan-matrix.outputs.matrix }}
-    steps:
-      - name: Checkout out Repo
-        uses: actions/checkout@v4
+    uses: ./.github/workflows/_get-image-list.yml
+    with:
+      examples: ${{ inputs.examples }}
+      images: ${{ inputs.images }}
 
-      - name: Set Matrix
-        id: scan-matrix
-        run: |
-          if [[ ! -z "${{ inputs.images }}" ]]; then
-              images=($(echo ${{ inputs.images }} | tr ',' ' '))
-              image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.')
-          else
-              pip install yq
-              examples=($(echo ${{ inputs.examples }} | tr ',' ' '))
-              image_list=[]
-              for example in ${examples[@]}
-              do
-                  images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.')
-                  image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
-              done
-          fi
-
-          echo "print image list..."
-          echo "$image_list" | jq . | jq -r '.[]'
-          echo "end of image list..."
-          echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
-
-  scan-license:
+  scan-docker:
     needs: get-image-list
-    if: ${{ fromJSON(inputs.sbom_scan) }} || ${{ fromJSON(inputs.trivy_scan) }}
     runs-on: "docker-build-${{ inputs.node }}"
     strategy:
       matrix:
         image: ${{ fromJson(needs.get-image-list.outputs.matrix) }}
       fail-fast: false
     steps:
+      - name: Clean up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/* || true
+          docker system prune -f
+
       - name: Pull Image
         run: |
           docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
           echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
 
       - name: SBOM Scan Container
         uses: anchore/[email protected]
-        if: always() && ${{ inputs.sbom_scan }}
+        if: ${{ inputs.sbom_scan }}
         with:
           image: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
           output-file: ${{ matrix.image }}-sbom-scan.txt
           format: 'spdx-json'
 
       - name: Security Scan Container
         uses: aquasecurity/[email protected]
-        if: always() && ${{ inputs.trivy_scan }}
+        if: ${{ inputs.trivy_scan }}
         with:
           image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
           output: ${{ matrix.image }}-trivy-scan.txt
@@ -104,18 +84,30 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+        run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} || true
+
+      - name: Collect Logs
+        if: always()
+        run: |
+          mkdir -p /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}
+          mv ${{ matrix.image }}-*-scan.txt /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}
 
+  upload-artifacts:
+    needs: scan-docker
+    runs-on: "docker-build-${{ inputs.node }}"
+    if: always()
+    steps:
       - uses: actions/[email protected]
-        if: always() && (${{ inputs.sbom_scan }}
         with:
-          name: sbom-scan
-          path: ${{ matrix.image }}-*-sbom-scan.txt
+          name: sbom-scan-${{ inputs.tag }}-${{ github.run_number }}
+          path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-sbom-scan.txt
           overwrite: true
 
       - uses: actions/[email protected]
-        if: always() && (${{ inputs.trivy_scan }}
         with:
-          name: trivy-scan
-          path: ${{ matrix.image }}-*-trivy-scan.txt
+          name: trivy-scan-${{ inputs.tag }}-${{ github.run_number }}
+          path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-trivy-scan.txt
           overwrite: true
+
+      - name: Remove Logs
+        run: rm -rf /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} && rm -rf /tmp/sbom-action-*