ci: update test env

.github/workflows/main.yaml

@@ -31,10 +31,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3
@@ -55,6 +55,8 @@ jobs:
         shell: bash
         run: |
           pip install -r requirements.txt --trusted-host https://test.pypi.org
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
 
       - name: Milvus deploy
 
@@ -164,10 +166,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3
@@ -188,6 +190,8 @@ jobs:
         shell: bash
         run: |
           pip install -r requirements.txt --trusted-host https://test.pypi.org
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
 
       - name: Milvus deploy
 
@@ -205,7 +209,7 @@ jobs:
         shell: bash
         run: |
           python example/prepare_data.py
-      
+
       - name: Upgrade Milvus
         timeout-minutes: 15
         shell: bash
@@ -249,10 +253,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3
@@ -273,6 +277,8 @@ jobs:
         shell: bash
         run: |
           pip install -r requirements.txt --trusted-host https://test.pypi.org
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
 
       - name: Milvus deploy
 
@@ -378,10 +384,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3
@@ -421,6 +427,8 @@ jobs:
         shell: bash
         run: |
           pip install -r requirements.txt --trusted-host https://test.pypi.org
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
       - name: Milvus deploy
 
         timeout-minutes: 15
@@ -494,6 +502,7 @@ jobs:
             deployment/${{ matrix.milvus_mode }}/logs
   test-backup-restore-api:
     runs-on: ubuntu-latest
+    needs: test-backup-restore-cli
     strategy:
       fail-fast: false
       matrix:
@@ -508,10 +517,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3
@@ -545,6 +554,8 @@ jobs:
         shell: bash
         run: |
           pip install -r requirements.txt --trusted-host https://test.pypi.org
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
 
       - name: Milvus deploy
 

.github/workflows/nightly.yaml

@@ -18,10 +18,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3

.github/workflows/perf.yaml

@@ -27,10 +27,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           cache: pip
 
       - uses: actions/setup-go@v3
@@ -62,6 +62,8 @@ jobs:
         shell: bash
         run: |
           pip install -r requirements.txt --trusted-host https://test.pypi.org
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
 
       - name: Milvus deploy
 

tests/base/client_base.py

@@ -398,7 +398,7 @@ def compare_collections(self, src_name, dist_name, output_fields=None, verify_by
         log.info(f"pk diff: {diff}")
         assert len(diff) == 0
         for i in range(len(src_res)):
-            assert src_res[i] == dist_res[i]
+            assert src_res[i] == dist_res[i], f"src: {src_res[i]}, dist: {dist_res[i]}"
         for coll in [collection_src, collection_dist]:
             try:
                 coll.release()

tests/requirements.txt

@@ -1,13 +1,9 @@
 --extra-index-url https://test.pypi.org/simple/
-loguru==0.5.3
-decorest==0.1.0
-typing_extensions==4.4.0
---extra-index-url https://test.pypi.org/simple/
 pytest-cov==2.8.1
 requests==2.26.0
-scikit-learn==1.1.2
+scikit-learn==1.1.3
 timeout_decorator==0.5.0
-ujson==4.2.0
+ujson==5.5.0
 pytest==7.2.0
 pytest-assume==2.4.3
 pytest-timeout==1.3.3
@@ -16,8 +12,8 @@ allure-pytest==2.7.0
 pytest-print==0.2.1
 pytest-level==0.1.1
 pytest-xdist==2.5.0
-pytest-loguru==0.2.0
-pymilvus==2.4.2
+pymilvus==2.4.5rc11
+pymilvus[bulk_writer]==2.4.5rc11
 pytest-rerunfailures==9.1.1
 git+https://github.com/Projectplace/pytest-tags
 ndg-httpsclient
@@ -30,22 +26,29 @@ PyYAML==6.0
 pytest-sugar==0.9.5
 pytest-parallel
 pytest-random-order
-
+numpy==1.25.0
 # for customize config test
 python-benedict==0.24.3
 timeout-decorator==0.5.0
+pytest-benchmark==4.0.0
 
-# version need to be consistent with protobuf used in pymilvus
-protobuf==3.20.0
-
-# for bulk load test
+# for bulk insert test
 minio==7.1.5
+npy-append-array==0.9.15
+Faker==19.2.0
 
-# for benchmark
-h5py==3.1.0
-pytest-benchmark==4.0.0
 
+
+# for log
+loguru==0.7.0
+
+# util
+psutil==5.9.4
+pandas==1.5.3
 tenacity==8.1.0
-# for brain float16 datatype
+
+
+
+# for bf16 datatype
 jax==0.4.13
 jaxlib==0.4.13

tests/testcases/test_restore_backup.py

@@ -838,7 +838,7 @@ def test_milvus_restore_back_with_upsert(self):
         output_fields = None
         self.compare_collections(name_origin, name_origin + suffix, output_fields=output_fields, verify_by_query=True)
 
-    @pytest.mark.tags(CaseLabel.L1)
+    @pytest.mark.tags(CaseLabel.MASTER)
     def test_milvus_restore_back_with_dup_pk(self):
         self._connect()
         name_origin = cf.gen_unique_str(prefix)