install torch cpu before install sentence_transformers to avoid nv …

…components (#221) * install torch cpu before install `sentence_transformers` to avoid nv components Signed-off-by: chensuyue <[email protected]> * add dataprep/redis/langchain test Signed-off-by: chensuyue <[email protected]> * add torch vision for retrievers langchain redis Signed-off-by: chensuyue <[email protected]> * expand test wait time Signed-off-by: chensuyue <[email protected]> * remove test Signed-off-by: chensuyue <[email protected]> * support torch cpu install Signed-off-by: chensuyue <[email protected]> * support torch cpu install Signed-off-by: chensuyue <[email protected]> * fix bug Signed-off-by: chensuyue <[email protected]> --------- Signed-off-by: chensuyue <[email protected]>
opea-project · Jun 24, 2024 · 2e3c032 · 2e3c032
1 parent 75eff63
commit 2e3c032
Show file tree

Hide file tree

Showing 10 changed files with 28 additions and 1 deletion.
diff --git a/.github/workflows/reuse-get-test-matrix.yml b/.github/workflows/reuse-get-test-matrix.yml
@@ -55,7 +55,7 @@ jobs:
               if [[ "${path_level_1[@]}" =~ "${service}" ]]; then
                   run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
               else
-                  vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md' | sort -u)
+                  vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md|*.sh' | sort -u)
                   for vendor in ${vendors}; do
                       if [[ "${path_level_3[@]}" =~ "${service}/${vendor}" ]]; then
                           sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u)

diff --git a/comps/dataprep/milvus/docker/Dockerfile b/comps/dataprep/milvus/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libgl1-mesa-glx \
@@ -21,6 +23,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/dataprep/milvus/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/dataprep/qdrant/docker/Dockerfile b/comps/dataprep/qdrant/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libgl1-mesa-glx \
@@ -21,6 +23,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/dataprep/qdrant/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/dataprep/redis/langchain/docker/Dockerfile b/comps/dataprep/redis/langchain/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libgl1-mesa-glx \
@@ -21,6 +23,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/dataprep/redis/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/dataprep/redis/langchain_ray/docker/Dockerfile b/comps/dataprep/redis/langchain_ray/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libgl1-mesa-glx \
@@ -21,6 +23,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/dataprep/redis/langchain_ray/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/dataprep/redis/llama_index/docker/Dockerfile b/comps/dataprep/redis/llama_index/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libgl1-mesa-glx \
@@ -21,6 +23,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/dataprep/redis/llama_index/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/embeddings/langchain/docker/Dockerfile b/comps/embeddings/langchain/docker/Dockerfile
@@ -4,6 +4,8 @@
 
 FROM langchain/langchain:latest
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
@@ -18,6 +20,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/embeddings/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/reranks/langchain/docker/Dockerfile b/comps/reranks/langchain/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
@@ -20,6 +22,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/reranks/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/retrievers/langchain/milvus/docker/Dockerfile b/comps/retrievers/langchain/milvus/docker/Dockerfile
@@ -6,6 +6,8 @@ FROM python:3.11-slim
 
 ENV LANG C.UTF-8
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libgl1-mesa-glx \
@@ -21,6 +23,7 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/milvus/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user

diff --git a/comps/retrievers/langchain/redis/docker/Dockerfile b/comps/retrievers/langchain/redis/docker/Dockerfile
@@ -4,6 +4,8 @@
 
 FROM langchain/langchain:latest
 
+ARG ARCH="cpu"
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
@@ -20,6 +22,7 @@ RUN chmod +x /home/user/comps/retrievers/langchain/redis/run.sh
 USER user
 
 RUN pip install --no-cache-dir --upgrade pip && \
+    if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/redis/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user