[PERF] increase delta conversion speed by batching

rust/benchmark/src/datasets/gist.rs

@@ -23,7 +23,7 @@ impl RecordDataset for GistDataset {
     async fn init() -> Result<Self> {
         // TODO(Sanket): Download file if it doesn't exist.
         // move file from downloads to cached path.
-        let current_path = "/Users/sanketkedia/Downloads/siftsmall/siftsmall_base.fvecs";
+        let current_path = "/Users/hammad/Downloads/siftsmall/siftsmall_base.fvecs";
         let base_file_path = get_or_populate_cached_dataset_file(
             "gist",
             "siftsmall_base.fvecs",

rust/blockstore/src/arrow/block/value/spann_posting_list_value.rs

@@ -115,14 +115,12 @@ impl ArrowWriteableValue for &SpannPostingList<'_> {
             inner_version_ref.append_value(doc_version);
         }
         let inner_embeddings_ref = builder.doc_embeddings_builder.values();
-        let mut f32_count = 0;
-        for embedding in doc_embeddings.into_iter() {
-            inner_embeddings_ref.values().append_value(embedding);
-            f32_count += 1;
-            if f32_count == embedding_dim {
-                inner_embeddings_ref.append(true);
-                f32_count = 0;
-            }
+        for chunk in doc_embeddings.chunks(embedding_dim) {
+            let validity = vec![true; chunk.len()];
+            inner_embeddings_ref
+                .values()
+                .append_values(chunk, &validity);
+            inner_embeddings_ref.append(true);
         }
         builder.doc_offset_ids_builder.append(true);
         builder.doc_versions_builder.append(true);

rust/worker/benches/spann.rs

@@ -278,14 +278,14 @@ fn bench_spann_compaction(c: &mut Criterion) {
                     "Added 9000 records to spann segment in {:?} ms",
                     start_time.elapsed().as_millis()
                 );
-                println!("Getting recall on 1000 records");
-                calculate_recall(
-                    &runtime,
-                    reader,
-                    &records[9000..10000],
-                    &records[0..9000],
-                    deleted_set,
-                );
+                // println!("Getting recall on 1000 records");
+                // calculate_recall(
+                //     &runtime,
+                //     reader,
+                //     &records[9000..10000],
+                //     &records[0..9000],
+                //     deleted_set,
+                // );
             })
         },
     );