add faiss

.github/workflows/run-opensearch-faiss-linux.yml

@@ -0,0 +1,56 @@
+name: Run OpenSearch (faiss) on Linux
+
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - run-opensearch.ipynb
+  pull_request:
+    branches:
+    - main
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Check Hardware Spec
+      run: /bin/bash ./scripts/get_hardware_info.sh
+    - name: Cache dataset
+      uses: actions/cache@v4
+      with:
+        path: |
+          dataset
+        key: ${{ runner.os }}-dataset
+    - name: Download data
+      run: /bin/bash ./scripts/setup.sh gha
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Cache pip modules
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Run ANN search
+      run: papermill run-opensearch.ipynb output-opensearch-linux.ipynb --log-output
+      env:
+        PRODUCT_VARIANT: "faiss"
+        SETTING_ENGINE: "faiss"
+    - name: 'Upload Results'
+      uses: actions/upload-artifact@v4
+      with:
+        name: opensearch-faiss-results
+        path: results.json
+

run-opensearch.ipynb

@@ -66,7 +66,7 @@
     "            \"bulk_size\": 1000,\n",
     "            \"index_name\": \"contents\",\n",
     "            \"engine\": \"lucene\", # faiss\"\n",
-    "            \"distance\": \"cosinesimil\", # \"innerproduct\"\n",
+    "            \"distance\": \"innerproduct\",\n",
     "            \"dimension\": 768,\n",
     "            \"hnsw_m\": 32,\n",
     "            \"hnsw_ef_construction\": 200,\n",
@@ -82,7 +82,7 @@
     "            \"bulk_size\": 1000,\n",
     "            \"index_name\": \"contents\",\n",
     "            \"engine\": \"lucene\", # faiss\"\n",
-    "            \"distance\": \"cosinesimil\", # \"innerproduct\"\n",
+    "            \"distance\": \"innerproduct\",\n",
     "            \"dimension\": 768,\n",
     "            \"hnsw_m\": 48,\n",
     "            \"hnsw_ef_construction\": 200,\n",
@@ -98,7 +98,7 @@
     "            \"bulk_size\": 1000,\n",
     "            \"index_name\": \"contents\",\n",
     "            \"engine\": \"lucene\", # faiss\"\n",
-    "            \"distance\": \"cosinesimil\", # \"innerproduct\"\n",
+    "            \"distance\": \"innerproduct\",\n",
     "            \"dimension\": 768,\n",
     "            \"hnsw_m\": 48,\n",
     "            \"hnsw_ef_construction\": 200,\n",
@@ -108,6 +108,7 @@
     "        },\n",
     "    }\n",
     "    target_setting = setting.get(target_name)\n",
+    "    target_setting[\"engine\"] = os.getenv(\"SETTING_ENGINE\", target_setting[\"engine\"])\n",
     "    target_setting[\"quantization\"] = os.getenv(\"SETTING_QUANTIZATION\", target_setting[\"quantization\"])\n",
     "    return DataSetConfig(**target_setting)\n"
    ]