Sync with main and resolve conflicts

Signed-off-by: Abolfazl Shahbazi <[email protected]>
opea-project · Sep 4, 2024 · 4ed3628 · 4ed3628
2 parents caa499a + 54aa943
commit 4ed3628
Show file tree

Hide file tree

Showing 212 changed files with 8,831 additions and 788 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,9 +1,21 @@
+/comps/agent/ [email protected]
+/comps/finetuning/ [email protected]
+/comps/guardrails/ [email protected]
 /comps/asr/ [email protected]
+/comps/intent_detection/ [email protected]
+/comps/knowledgegraphs/ [email protected]
 /comps/cores/ [email protected]
 /comps/dataprep/ [email protected]
 /comps/embeddings/ [email protected]
 /comps/guardrails/ [email protected]
 /comps/llms/ [email protected]
+/comps/lvms/ [email protected]
+/comps/nginx/ [email protected]
+/comps/prompt_registry/ [email protected]
+/comps/chathistory/ [email protected]
 /comps/reranks/ [email protected]
 /comps/retrievers/ [email protected]
 /comps/tts/ [email protected]
+/comps/ragas/ [email protected]
+/comps/vectorstores/ [email protected]
+/comps/web_retrievers/ [email protected]
diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml
@@ -20,17 +20,26 @@ on:
         default: true
         required: false
         type: boolean
-      # scan:
-      #   default: true
-      #   required: false
-      #   type: boolean
+      test:
+        default: true
+        description: "Test comps with docker compose"
+        required: false
+        type: boolean
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI or CD"
+        required: false
+        type: string
+
 jobs:
   ####################################################################################################
   # Image Build
   ####################################################################################################
   build-images:
     runs-on: "docker-build-${{ inputs.node }}"
     continue-on-error: true
+    outputs:
+      file_exists: ${{ steps.get-yaml-path.outputs.file_exists }}
     steps:
       - name: Clean Up Working Directory
         run: sudo rm -rf ${{github.workspace}}/*
@@ -39,82 +48,46 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Clone required Repo
+        id: get-yaml-path
         run: |
           cd ${{ github.workspace }}/.github/workflows/docker/compose
           # service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
-          docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
-          echo ${docker_compose_yml}
+          if [[ "${{ inputs.mode }}" == "CD" ]]; then
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
+          else
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          fi
+          echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
+          if [ -f "$docker_compose_yml" ]; then
+            echo "file_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "There is no ${{ inputs.mode }} part of ${{ inputs.service }} that needs to be executed."
+            echo "file_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          cd ${{ github.workspace }}
           if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
-              git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
+              git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 && cd ../
           fi
           if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/vllm-project/vllm.git vllm-openvino
           fi
-          # echo "service=$service" >> $GITHUB_ENV
 
       - name: Build Image
-        if: ${{ fromJSON(inputs.build) }}
+        if: ${{ fromJSON(inputs.build) && steps.get-yaml-path.outputs.file_exists == 'true' }}
         uses: opea-project/validation/actions/image-build@main
         with:
-          work_dir: ${{ github.workspace }}/
-          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
           registry: ${OPEA_IMAGE_REPO}opea
           tag: ${{ inputs.tag }}
-  # ####################################################################################################
-  # # Trivy Scan
-  # ####################################################################################################
-  #   get-image-list:
-  #     needs: [build-images]
-  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
-  #     runs-on: ubuntu-latest
-  #     outputs:
-  #       matrix: ${{ steps.scan-matrix.outputs.matrix }}
-  #     steps:
-  #       - name: Checkout out Repo
-  #         uses: actions/checkout@v4
-
-  #       - name: Set Matrix
-  #         id: scan-matrix
-  #         run: |
-  #           pip install yq
-  #           compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
-  #           echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
-
-  #   scan-images:
-  #     needs: [get-image-list, build-images]
-  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
-  #     runs-on: "docker-build-${{ inputs.node }}"
-  #     strategy:
-  #       matrix:
-  #         image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
-  #       fail-fast: false
-  #     steps:
-  #       - name: Pull Image
-  #         run: |
-  #           docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-  #           echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
-
-  #       - name: Scan Container
-  #         uses: opea-project/validation/actions/trivy-scan@main
-  #         with:
-  #           image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
-  #           output: ${{ matrix.image }}-scan.txt
-
-  #       - name: Cleanup
-  #         if: always()
-  #         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-
-  #       - uses: actions/[email protected]
-  #         with:
-  #           name: ${{ matrix.image }}-scan
-  #           path: ${{ matrix.image }}-scan.txt
-  #           overwrite: true
 
   ####################################################################################################
   # Docker Compose Test
   ####################################################################################################
   test-service-compose:
     needs: [build-images]
+    if: ${{ fromJSON(inputs.test) && needs.build-images.outputs.file_exists == 'true' }}
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       tag: ${{ inputs.tag }}

diff --git a/.github/workflows/_get-image-list.yml b/.github/workflows/_get-image-list.yml
@@ -0,0 +1,65 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Get Image List
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      services:
+        default: ""
+        required: false
+        type: string
+      images:
+        default: ""
+        required: false
+        type: string
+      mode:
+        default: "CD"
+        required: false
+        type: string
+    outputs:
+      matrix:
+        description: "Image List"
+        value: ${{ jobs.get-image-list.outputs.matrix }}
+
+jobs:
+  get-image-list:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Set Matrix
+        id: get-matrix
+        run: |
+          image_list=[]
+          if [[ ! -z "${{ inputs.services }}" ]]; then
+              pip install yq
+              services=($(echo ${{ inputs.services }} | tr ',' ' '))
+              for service in ${services[@]}
+              do
+                  if [[ "${{ inputs.mode }}" == "CD" ]]; then
+                      docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose-cd.yaml
+                  else
+                      docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
+                  fi
+                  if [ -f "$docker_compose_yml" ]; then
+                      images=$(cat $docker_compose_yml | yq -r '.[]' | jq 'keys' | jq -c '.')
+                      image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
+                  fi
+              done
+          fi
+
+          if [[ ! -z "${{ inputs.images }}" ]]; then
+              images=($(echo ${{ inputs.images }} | tr ',' ' '))
+              input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.')
+              image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list}))
+          fi
+
+          echo "print image list..."
+          echo "$image_list" | jq . | jq -r '.[]'
+          echo "end of image list..."
+          echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/_get-test-matrix.yml b/.github/workflows/_get-test-matrix.yml
@@ -25,7 +25,6 @@ jobs:
           else
             echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
           fi
-          echo "checkout ref ${{ env.CHECKOUT_REF }}"
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -39,7 +38,7 @@ jobs:
           set -xe
           if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
             LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=main" | jq -r '.[0].sha')
+            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
             echo "Latest commit SHA is $LATEST_COMMIT_SHA"
             base_commit=$LATEST_COMMIT_SHA
           else

diff --git a/.github/workflows/docker/compose/agent-compose-cd.yaml b/.github/workflows/docker/compose/agent-compose-cd.yaml
@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  comps-agent-langchain:
+    build:
+      dockerfile: comps/agent/langchain/docker/Dockerfile
+    image: ${REGISTRY:-opea}/comps-agent-langchain:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/asr-compose.yaml b/.github/workflows/docker/compose/asr-compose.yaml
@@ -7,12 +7,12 @@ services:
   asr:
     build:
       dockerfile: comps/asr/Dockerfile
-    image: ${REGISTRY}opea/asr:${TAG:-latest}
+    image: ${REGISTRY:-opea}/asr:${TAG:-latest}
   whisper:
     build:
       dockerfile: comps/asr/whisper/Dockerfile
-    image: ${REGISTRY}opea/whisper:${TAG:-latest}
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
   whisper-gaudi:
     build:
       dockerfile: comps/asr/whisper/Dockerfile_hpu
-    image: ${REGISTRY}opea/whisper-gaudi:${TAG:-latest}
+    image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/chathistory-compose-cd.yaml b/.github/workflows/docker/compose/chathistory-compose-cd.yaml
@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  chathistory-mongo-server:
+    build:
+      dockerfile: comps/chathistory/mongo/docker/Dockerfile
+    image: ${REGISTRY:-opea}/chathistory-mongo-server:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml
@@ -0,0 +1,25 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# others: dataprep-redis-llama-index,dataprep-on-ray-redis
+services:
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
+  dataprep-milvus:
+    build:
+      dockerfile: comps/dataprep/milvus/docker/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
+  dataprep-pgvector:
+    build:
+      dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
+  dataprep-pinecone:
+    build:
+      dockerfile: comps/dataprep/pinecone/docker/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml
@@ -8,16 +8,8 @@ services:
   dataprep-redis:
     build:
       dockerfile: comps/dataprep/redis/langchain/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-redis:${TAG:-latest}
+    image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
   dataprep-qdrant:
     build:
       dockerfile: comps/dataprep/qdrant/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
-  dataprep-redis-llama-index:
-    build:
-      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
+    image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml
@@ -0,0 +1,16 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  embedding-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-langchain-mosec-endpoint:${TAG:-latest}
+  embedding-langchain-mosec:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-langchain-mosec:${TAG:-latest}
+  embedding-tei-llama-index:
+    build:
+      dockerfile: comps/embeddings/llama_index/docker/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/embeddings-compose.yaml b/.github/workflows/docker/compose/embeddings-compose.yaml
@@ -7,4 +7,4 @@ services:
   embedding-tei:
     build:
       dockerfile: comps/embeddings/langchain/docker/Dockerfile
-    image: ${REGISTRY}opea/embedding-tei:${TAG:-latest}
+    image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/guardrails-compose-cd.yaml b/.github/workflows/docker/compose/guardrails-compose-cd.yaml
@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  guardrails-pii-detection:
+    build:
+      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/guardrails-compose.yaml b/.github/workflows/docker/compose/guardrails-compose.yaml
@@ -8,8 +8,4 @@ services:
   guardrails-tgi:
     build:
       dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
-    image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest}
-  guardrails-pii-detection:
-    build:
-      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
-    image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}
+    image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml
@@ -0,0 +1,13 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  llm-native:
+    build:
+      dockerfile: comps/llms/text-generation/native/docker/Dockerfile
+    image: ${REGISTRY:-opea}/llm-native:${TAG:-latest}
+  vllm-openvino:
+    build:
+      context: vllm-openvino
+      dockerfile: Dockerfile.openvino
+    image: ${REGISTRY:-opea}/vllm-openvino:${TAG:-latest}