Skip to content

Commit

Permalink
update image build yaml (#529)
Browse files Browse the repository at this point in the history
Signed-off-by: chensuyue <[email protected]>
Signed-off-by: zepan <[email protected]>
  • Loading branch information
ZePan110 authored Aug 30, 2024
1 parent ed44b44 commit b541fd8
Show file tree
Hide file tree
Showing 19 changed files with 201 additions and 84 deletions.
79 changes: 21 additions & 58 deletions .github/workflows/_comps-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,17 @@ on:
default: true
required: false
type: boolean
# scan:
# default: true
# required: false
# type: boolean
test:
default: true
description: "Test comps with docker compose"
required: false
type: boolean
mode:
default: "CD"
description: "Whether the test range is CI or CD"
required: false
type: string

jobs:
####################################################################################################
# Image Build
Expand All @@ -39,82 +46,38 @@ jobs:
uses: actions/checkout@v4

- name: Clone required Repo
id: get-yaml-path
run: |
cd ${{ github.workspace }}/.github/workflows/docker/compose
# service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
echo ${docker_compose_yml}
if [[ "${{ inputs.mode }}" == "CD" ]]; then
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
else
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
fi
echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
fi
if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
git clone https://github.com/vllm-project/vllm.git vllm-openvino
fi
# echo "service=$service" >> $GITHUB_ENV
- name: Build Image
if: ${{ fromJSON(inputs.build) }}
uses: opea-project/validation/actions/image-build@main
with:
work_dir: ${{ github.workspace }}/
docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
work_dir: ${{ github.workspace }}
docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
registry: ${OPEA_IMAGE_REPO}opea
tag: ${{ inputs.tag }}
# ####################################################################################################
# # Trivy Scan
# ####################################################################################################
# get-image-list:
# needs: [build-images]
# if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
# runs-on: ubuntu-latest
# outputs:
# matrix: ${{ steps.scan-matrix.outputs.matrix }}
# steps:
# - name: Checkout out Repo
# uses: actions/checkout@v4

# - name: Set Matrix
# id: scan-matrix
# run: |
# pip install yq
# compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
# echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT

# scan-images:
# needs: [get-image-list, build-images]
# if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
# runs-on: "docker-build-${{ inputs.node }}"
# strategy:
# matrix:
# image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
# fail-fast: false
# steps:
# - name: Pull Image
# run: |
# docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
# echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV

# - name: Scan Container
# uses: opea-project/validation/actions/trivy-scan@main
# with:
# image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
# output: ${{ matrix.image }}-scan.txt

# - name: Cleanup
# if: always()
# run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}

# - uses: actions/[email protected]
# with:
# name: ${{ matrix.image }}-scan
# path: ${{ matrix.image }}-scan.txt
# overwrite: true

####################################################################################################
# Docker Compose Test
####################################################################################################
test-service-compose:
needs: [build-images]
if: ${{ fromJSON(inputs.test) }}
uses: ./.github/workflows/_run-docker-compose.yml
with:
tag: ${{ inputs.tag }}
Expand Down
9 changes: 9 additions & 0 deletions .github/workflows/docker/compose/agent-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
comps-agent-langchain:
build:
dockerfile: comps/agent/langchain/docker/Dockerfile
image: ${REGISTRY}opea/comps-agent-langchain:${TAG:-latest}
9 changes: 9 additions & 0 deletions .github/workflows/docker/compose/chathistory-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
chathistory-mongo-server:
build:
dockerfile: comps/chathistory/mongo/docker/Dockerfile
image: ${REGISTRY}opea/chathistory-mongo-server:${TAG:-latest}
25 changes: 25 additions & 0 deletions .github/workflows/docker/compose/dataprep-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# others: dataprep-redis-llama-index,dataprep-on-ray-redis
services:
dataprep-redis-llama-index:
build:
dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
dataprep-on-ray-redis:
build:
dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
dataprep-milvus:
build:
dockerfile: comps/dataprep/milvus/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-milvus:${TAG:-latest}
dataprep-pgvector:
build:
dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-pgvector:${TAG:-latest}
dataprep-pinecone:
build:
dockerfile: comps/dataprep/pinecone/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-pinecone:${TAG:-latest}
8 changes: 0 additions & 8 deletions .github/workflows/docker/compose/dataprep-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,3 @@ services:
build:
dockerfile: comps/dataprep/qdrant/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
dataprep-redis-llama-index:
build:
dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
dataprep-on-ray-redis:
build:
dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
16 changes: 16 additions & 0 deletions .github/workflows/docker/compose/embeddings-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
embedding-langchain-mosec-endpoint:
build:
dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
image: ${REGISTRY}opea/embedding-langchain-mosec-endpoint:${TAG:-latest}
embedding-langchain-mosec:
build:
dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
image: ${REGISTRY}opea/embedding-langchain-mosec:${TAG:-latest}
embedding-tei-llama-index:
build:
dockerfile: comps/embeddings/llama_index/docker/Dockerfile
image: ${REGISTRY}opea/embedding-tei-llama-index:${TAG:-latest}
8 changes: 8 additions & 0 deletions .github/workflows/docker/compose/guardrails-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
guardrails-pii-detection:
build:
dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}
4 changes: 0 additions & 4 deletions .github/workflows/docker/compose/guardrails-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,3 @@ services:
build:
dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest}
guardrails-pii-detection:
build:
dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}
8 changes: 8 additions & 0 deletions .github/workflows/docker/compose/llms-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
llm-native:
build:
dockerfile: comps/llms/text-generation/native/docker/Dockerfile
image: ${REGISTRY}opea/llm-native:${TAG:-latest}
23 changes: 23 additions & 0 deletions .github/workflows/docker/compose/lvms-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
lvm:
build:
dockerfile: comps/lvms/Dockerfile
image: ${REGISTRY}opea/lvm:${TAG:-latest}
# Xeon CPU
llava:
build:
dockerfile: comps/lvms/llava/Dockerfile
image: ${REGISTRY}opea/llava:${TAG:-latest}
# Gaudi2 HPU
llava_hpu:
build:
dockerfile: comps/lvms/llava/Dockerfile_hpu
image: ${REGISTRY}opea/llava_hpu:${TAG:-latest}
lvm-tgi:
build:
dockerfile: comps/lvms/Dockerfile_tgi
image: ${REGISTRY}opea/lvm-tgi:${TAG:-latest}
9 changes: 9 additions & 0 deletions .github/workflows/docker/compose/nginx-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
nginx:
build:
dockerfile: comps/nginx/docker/Dockerfile
image: ${REGISTRY}opea/nginx:${TAG:-latest}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
promptregistry-mongo-server:
build:
dockerfile: comps/prompt_registry/mongo/docker/Dockerfile
image: ${REGISTRY}opea/promptregistry-mongo-server:${TAG:-latest}
16 changes: 16 additions & 0 deletions .github/workflows/docker/compose/reranks-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
reranking-fastrag:
build:
dockerfile: comps/reranks/fastrag/docker/Dockerfile
image: ${REGISTRY}opea/reranking-fastrag:${TAG:-latest}
reranking-langchain-mosec-endpoint:
build:
dockerfile: comps/reranks/langchain-mosec/mosec-docker/Dockerfile
image: ${REGISTRY}opea/reranking-langchain-mosec-endpoint:${TAG:-latest}
reranking-langchain-mosec:
build:
dockerfile: comps/reranks/langchain-mosec/docker/Dockerfile
image: ${REGISTRY}opea/reranking-langchain-mosec:${TAG:-latest}
16 changes: 16 additions & 0 deletions .github/workflows/docker/compose/retrievers-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,19 @@ services:
build:
dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile
image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}
retriever-pgvector:
build:
dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile
image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}
retriever-pinecone:
build:
dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile
image: ${REGISTRY}opea/retriever-pinecone:${TAG:-latest}
retriever-milvus:
build:
dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile
image: ${REGISTRY}opea/retriever-milvus:${TAG:-latest}
retriever-redis-llamaindex:
build:
dockerfile: comps/retrievers/llamaindex/docker/Dockerfile
image: ${REGISTRY}opea/retriever-redis-llamaindex:${TAG:-latest}
21 changes: 19 additions & 2 deletions .github/workflows/manual-comps-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,29 @@ on:
inputs:
services:
default: "asr"
description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,llms,lvms,nginx,prompt_registry,reranks,retrievers,tts,web_retrievers]"
required: true
type: string
build:
default: true
description: "Build test required images for Comps"
required: false
type: boolean
test:
default: true
description: "Test comps with docker compose"
required: false
type: boolean
tag:
default: "comps"
description: "Tag to apply to images"
required: true
type: string
mode:
default: "CD"
description: "Whether the test range is CI or CD"
required: false
type: string

permissions: read-all

Expand All @@ -40,6 +55,8 @@ jobs:
uses: ./.github/workflows/_comps-workflow.yml
with:
service: ${{ matrix.service }}
tag: "comps"
tag: ${{ inputs.tag }}
node: gaudi
mode: ${{ inputs.mode }}
test: ${{ inputs.test }}
secrets: inherit
4 changes: 2 additions & 2 deletions comps/nginx/docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@ FROM nginx:alpine

RUN apk add --no-cache gettext

COPY nginx.conf.template /etc/nginx/nginx.conf.template
COPY comps/nginx/docker/nginx.conf.template /etc/nginx/nginx.conf.template

ENV FRONTEND_SERVICE_IP=localhost
ENV FRONTEND_SERVICE_PORT=5173
ENV BACKEND_SERVICE_NAME=chatqna
ENV BACKEND_SERVICE_IP=localhost
ENV BACKEND_SERVICE_PORT=8888

COPY start-nginx.sh /usr/local/bin/start-nginx.sh
COPY comps/nginx/docker/start-nginx.sh /usr/local/bin/start-nginx.sh
RUN chmod +x /usr/local/bin/start-nginx.sh

CMD ["/usr/local/bin/start-nginx.sh"]
1 change: 1 addition & 0 deletions comps/retrievers/llamaindex/docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
FROM ubuntu:22.04

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
python3-pip \
libgl1-mesa-glx \
libjemalloc-dev

Expand Down
Loading

0 comments on commit b541fd8

Please sign in to comment.