diff --git a/.github/workflows/reuse-get-test-matrix.yml b/.github/workflows/reuse-get-test-matrix.yml index 3a7704b48..58cddb25b 100644 --- a/.github/workflows/reuse-get-test-matrix.yml +++ b/.github/workflows/reuse-get-test-matrix.yml @@ -55,7 +55,7 @@ jobs: if [[ "${path_level_1[@]}" =~ "${service}" ]]; then run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"}," else - vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md' | sort -u) + vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md|*.sh' | sort -u) for vendor in ${vendors}; do if [[ "${path_level_3[@]}" =~ "${service}/${vendor}" ]]; then sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u) diff --git a/comps/dataprep/milvus/docker/Dockerfile b/comps/dataprep/milvus/docker/Dockerfile index f2f9038c0..4b142fc4e 100644 --- a/comps/dataprep/milvus/docker/Dockerfile +++ b/comps/dataprep/milvus/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ libgl1-mesa-glx \ @@ -21,6 +23,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/dataprep/milvus/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/dataprep/qdrant/docker/Dockerfile b/comps/dataprep/qdrant/docker/Dockerfile index 45d85c269..5297d6101 100644 --- a/comps/dataprep/qdrant/docker/Dockerfile +++ b/comps/dataprep/qdrant/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ libgl1-mesa-glx \ @@ -21,6 +23,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/dataprep/qdrant/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/dataprep/redis/langchain/docker/Dockerfile b/comps/dataprep/redis/langchain/docker/Dockerfile index 215386750..25052816a 100644 --- a/comps/dataprep/redis/langchain/docker/Dockerfile +++ b/comps/dataprep/redis/langchain/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ libgl1-mesa-glx \ @@ -21,6 +23,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip setuptools && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/dataprep/redis/langchain/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/dataprep/redis/langchain_ray/docker/Dockerfile b/comps/dataprep/redis/langchain_ray/docker/Dockerfile index aec2506ec..1fec0fa07 100644 --- a/comps/dataprep/redis/langchain_ray/docker/Dockerfile +++ b/comps/dataprep/redis/langchain_ray/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ libgl1-mesa-glx \ @@ -21,6 +23,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip setuptools && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/dataprep/redis/langchain_ray/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/dataprep/redis/llama_index/docker/Dockerfile b/comps/dataprep/redis/llama_index/docker/Dockerfile index 1735c0fcf..14a4849de 100644 --- a/comps/dataprep/redis/llama_index/docker/Dockerfile +++ b/comps/dataprep/redis/llama_index/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ libgl1-mesa-glx \ @@ -21,6 +23,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip setuptools && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/dataprep/redis/llama_index/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/embeddings/langchain/docker/Dockerfile b/comps/embeddings/langchain/docker/Dockerfile index cf27d6819..464bacf66 100644 --- a/comps/embeddings/langchain/docker/Dockerfile +++ b/comps/embeddings/langchain/docker/Dockerfile @@ -4,6 +4,8 @@ FROM langchain/langchain:latest +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ libjemalloc-dev \ @@ -18,6 +20,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/embeddings/langchain/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/reranks/langchain/docker/Dockerfile b/comps/reranks/langchain/docker/Dockerfile index dbf252e1d..2daa22698 100644 --- a/comps/reranks/langchain/docker/Dockerfile +++ b/comps/reranks/langchain/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ libjemalloc-dev \ @@ -20,6 +22,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/reranks/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/retrievers/langchain/milvus/docker/Dockerfile b/comps/retrievers/langchain/milvus/docker/Dockerfile index 6d161be6b..69a971184 100644 --- a/comps/retrievers/langchain/milvus/docker/Dockerfile +++ b/comps/retrievers/langchain/milvus/docker/Dockerfile @@ -6,6 +6,8 @@ FROM python:3.11-slim ENV LANG C.UTF-8 +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ libgl1-mesa-glx \ @@ -21,6 +23,7 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/milvus/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user diff --git a/comps/retrievers/langchain/redis/docker/Dockerfile b/comps/retrievers/langchain/redis/docker/Dockerfile index 990ca4ebc..ee360269c 100644 --- a/comps/retrievers/langchain/redis/docker/Dockerfile +++ b/comps/retrievers/langchain/redis/docker/Dockerfile @@ -4,6 +4,8 @@ FROM langchain/langchain:latest +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ libjemalloc-dev \ @@ -20,6 +22,7 @@ RUN chmod +x /home/user/comps/retrievers/langchain/redis/run.sh USER user RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/redis/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home/user