feat: Add BQ to CI benchmarks (#148)

* feat: Add BQ to CI benchmarks * feat: Allow manual benchmarks to specify engine config
qdrant · May 31, 2024 · 9598214 · 9598214
1 parent a3a9b6b
commit 9598214
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 0 deletions.
diff --git a/.github/workflows/continuous-benchmark.yaml b/.github/workflows/continuous-benchmark.yaml
@@ -26,6 +26,7 @@ jobs:
             DATASET_TO_ENGINE["laion-small-clip"]="qdrant-continuous-benchmark"
             DATASET_TO_ENGINE["msmarco-sparse-100K"]="qdrant-sparse-vector"
             DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark"
+            DATASET_TO_ENGINE["dbpedia-openai-1M-1536-angular"]="qdrant-bq-continuous-benchmark"
 
             for dataset in "${!DATASET_TO_ENGINE[@]}"; do
               export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}

diff --git a/.github/workflows/manual-benchmark.yaml b/.github/workflows/manual-benchmark.yaml
@@ -10,6 +10,9 @@ on:
         dataset:
             description: "Dataset to benchmark"
             default: laion-small-clip
+        engine_config:
+            description: "Engine config to benchmark"
+            default: qdrant-continuous-benchmark
 
 jobs:
   runManualBenchmark:
@@ -27,6 +30,7 @@ jobs:
           export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
           export QDRANT_VERSION=${{ inputs.qdrant_version }}
           export DATASETS=${{ inputs.dataset }}
+          export ENGINE_NAME=${{ inputs.engine_config }}
           export POSTGRES_TABLE=benchmark_manual
           bash -x tools/setup_ci.sh
           bash -x tools/run_ci.sh
diff --git a/experiments/configurations/qdrant-single-node.json b/experiments/configurations/qdrant-single-node.json
@@ -45,6 +45,33 @@
     ],
     "upload_params": { "parallel": 16, "batch_size": 1024 }
   },
+  {
+    "name": "qdrant-bq-continuous-benchmark",
+    "engine": "qdrant",
+    "connection_params": { "timeout": 30 },
+    "collection_params": {
+      "hnsw_config": {
+        "m": 32,
+        "ef_construct": 256
+      },
+      "quantization_config": { "binary": {"always_ram": true} },
+      "optimizers_config": {
+        "max_segment_size": 1000000,
+        "default_segment_number": 3,
+        "memmap_threshold": 10000000
+      }
+    },
+    "search_params": [
+      {
+        "parallel": 8,
+        "config": {
+          "hnsw_ef": 256,
+          "quantization": { "rescore": true, "oversampling": 2.0 }
+        }
+      }
+    ],
+    "upload_params": { "parallel": 16, "batch_size": 1024 }
+  },
   {
     "name": "qdrant-sparse-vector",
     "engine": "qdrant",