diff --git a/src/VecSim/spaces/computer/preprocessor_container.h b/src/VecSim/spaces/computer/preprocessor_container.h index cd2f0dec8..e9ed08f8a 100644 --- a/src/VecSim/spaces/computer/preprocessor_container.h +++ b/src/VecSim/spaces/computer/preprocessor_container.h @@ -171,10 +171,17 @@ MultiPreprocessorsContainer::preprocess(const void *o void *storage_blob = nullptr; void *query_blob = nullptr; + + // Use of separate variables for the storage_blob_size and query_blob_size, in case we need to + // change their sizes to different values. + size_t storage_blob_size = input_blob_size; + size_t query_blob_size = input_blob_size; + for (auto pp : preprocessors) { if (!pp) break; - pp->preprocess(original_blob, storage_blob, query_blob, input_blob_size, this->alignment); + pp->preprocess(original_blob, storage_blob, query_blob, storage_blob_size, query_blob_size, + this->alignment); } // At least one blob was allocated. diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 88c34506e..f981b15a8 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -12,6 +12,7 @@ #include #include #include +#include #include "VecSim/memory/vecsim_base.h" #include "VecSim/spaces/spaces.h" @@ -23,8 +24,12 @@ class PreprocessorInterface : public VecsimBaseObject { : VecsimBaseObject(allocator) {} // Note: input_blob_size is relevant for both storage blob and query blob, as we assume results // are the same size. + // Use the the overload below for different sizes. virtual void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const = 0; + virtual void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const = 0; virtual void preprocessForStorage(const void *original_blob, void *&storage_blob, size_t &input_blob_size) const = 0; virtual void preprocessQuery(const void *original_blob, void *&query_blob, @@ -44,6 +49,20 @@ class CosinePreprocessor : public PreprocessorInterface { : PreprocessorInterface(allocator), normalize_func(spaces::GetNormalizeFunc()), dim(dim), processed_bytes_count(processed_bytes_count) {} + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + // This assert verifies that that the current use of this function is for blobs of the same + // size, which is the case for the Cosine preprocessor. If we ever need to support different + // sizes for storage and query blobs, we can remove the assert and implement the logic to + // handle different sizes. + assert(storage_blob_size == query_blob_size); + + preprocess(original_blob, storage_blob, query_blob, storage_blob_size, alignment); + // Ensure both blobs have the same size after processing. + query_blob_size = storage_blob_size; + } + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const override { // This assert verifies that if a blob was allocated by a previous preprocessor, its @@ -128,3 +147,169 @@ class CosinePreprocessor : public PreprocessorInterface { const size_t dim; const size_t processed_bytes_count; }; + +/* + * QuantPreprocessor is a preprocessor that quantizes the input vector of INPUT_TYPE (float) to a + * lower precision representation using OUTPUT_TYPE (uint8_t). It stores the quantized values along + * with metadata (min value and scaling factor) in a single contiguous blob. The quantized values + * are then stored in an OUTPUT_TYPE array. The quantization is done by finding the minimum and + * maximum values of the input vector, and then scaling the values to fit in the range of [0, 255]. + * The quantized blob size is: dim_elements * sizeof(OUTPUT_TYPE) + 2 * sizeof(float) + */ +class QuantPreprocessor : public PreprocessorInterface { + using INPUT_TYPE = float; + using OUTPUT_TYPE = uint8_t; + +public: + // Constructor for backward compatibility (single blob size) + QuantPreprocessor(std::shared_ptr allocator, size_t dim) + : PreprocessorInterface(allocator), dim(dim), + storage_bytes_count(dim * sizeof(OUTPUT_TYPE) + 2 * sizeof(float)) { + } // quantized + min + delta + + // Helper function to perform quantization. This function is used by both preprocess and + // preprocessQuery and supports in-place quantization of the storage blob. + void quantize(const INPUT_TYPE *input, OUTPUT_TYPE *quantized) const { + assert(input && quantized); + // Find min and max values + auto [min_val, max_val] = find_min_max(input); + + // Calculate scaling factor + const float diff = (max_val - min_val); + const float delta = diff == 0.0f ? 1.0f : diff / 255.0f; + const float inv_delta = 1.0f / delta; + + // Quantize the values + for (size_t i = 0; i < this->dim; i++) { + quantized[i] = static_cast(std::round((input[i] - min_val) * inv_delta)); + } + + float *metadata = reinterpret_cast(quantized + this->dim); + + // Store min_val, delta, in the metadata + metadata[0] = min_val; + metadata[1] = delta; + } + + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &input_blob_size, unsigned char alignment) const override { + // For backward compatibility - delegate to the two-size version with identical sizes + preprocess(original_blob, storage_blob, query_blob, input_blob_size, input_blob_size, + alignment); + } + + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + // CASE 1: STORAGE BLOB NEEDS ALLOCATION + if (!storage_blob) { + // Allocate aligned memory for the quantized storage blob + storage_blob = static_cast( + this->allocator->allocate_aligned(this->storage_bytes_count, alignment)); + + // Quantize directly from original data + const INPUT_TYPE *input = static_cast(original_blob); + quantize(input, static_cast(storage_blob)); + } + // CASE 2: STORAGE BLOB EXISTS + else { + // CASE 2A: STORAGE AND QUERY SHARE MEMORY + if (storage_blob == query_blob) { + // Need to allocate a separate storage blob since query remains float32 + // while storage needs to be quantized + void *new_storage = + this->allocator->allocate_aligned(this->storage_bytes_count, alignment); + + // Quantize from the shared blob (query_blob) to the new storage blob + quantize(static_cast(query_blob), + static_cast(new_storage)); + + // Update storage_blob to point to the new memory + storage_blob = new_storage; + } + // CASE 2B: SEPARATE STORAGE AND QUERY BLOBS + else { + // Check if storage blob needs resizing + if (storage_blob_size < this->storage_bytes_count) { + // Allocate new storage with correct size + OUTPUT_TYPE *new_storage = static_cast( + this->allocator->allocate_aligned(this->storage_bytes_count, alignment)); + + // Quantize from old storage to new storage + quantize(static_cast(storage_blob), + static_cast(new_storage)); + + // Free old storage and update pointer + this->allocator->free_allocation(storage_blob); + storage_blob = new_storage; + } else { + // Storage blob is large enough, quantize in-place + quantize(static_cast(storage_blob), + static_cast(storage_blob)); + } + } + } + + storage_blob_size = this->storage_bytes_count; + } + + void preprocessForStorage(const void *original_blob, void *&blob, + size_t &input_blob_size) const override { + // Allocate quantized blob if needed + if (!blob) { + blob = this->allocator->allocate(storage_bytes_count); + } + + // Cast to appropriate types + const INPUT_TYPE *input = static_cast(original_blob); + OUTPUT_TYPE *quantized = static_cast(blob); + quantize(input, quantized); + + input_blob_size = storage_bytes_count; + } + + void preprocessQuery(const void *original_blob, void *&blob, size_t &query_blob_size, + unsigned char alignment) const override { + // No-op: queries remain as float32 + } + + void preprocessQueryInPlace(void *blob, size_t input_blob_size, + unsigned char alignment) const override { + // No-op: queries remain as float32 + } + + void preprocessStorageInPlace(void *original_blob, size_t input_blob_size) const override { + assert(original_blob); + assert(input_blob_size >= storage_bytes_count && + "Input buffer too small for in-place quantization"); + + quantize(static_cast(original_blob), + static_cast(original_blob)); + } + +private: + std::pair find_min_max(const INPUT_TYPE *input) const { + float min_val = input[0]; + float max_val = input[0]; + + size_t i = 1; + // Process 4 elements at a time for better performance + for (; i + 3 < dim; i += 4) { + const float v0 = input[i]; + const float v1 = input[i + 1]; + const float v2 = input[i + 2]; + const float v3 = input[i + 3]; + min_val = std::min({min_val, v0, v1, v2, v3}); + max_val = std::max({max_val, v0, v1, v2, v3}); + } + // Handle remaining elements + for (; i < dim; i++) { + min_val = std::min(min_val, input[i]); + max_val = std::max(max_val, input[i]); + } + return {min_val, max_val}; + } + + const size_t dim; + const size_t storage_bytes_count; +}; diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index f6eabd3c8..917167e41 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -10,6 +10,7 @@ #include "gtest/gtest.h" #include "VecSim/vec_sim.h" #include "VecSim/spaces/computer/preprocessor_container.h" +#include "VecSim/spaces/computer/preprocessors.h" #include "VecSim/spaces/computer/calculator.h" #include "unit_test_utils.h" #include "tests_utils.h" @@ -65,7 +66,16 @@ class DummyStoragePreprocessor : public PreprocessorInterface { if (!value_to_add_query) value_to_add_query = value_to_add_storage; } - + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + // This assert verifies that there's no use for this function for now - different sizes for + // storage and query blobs. If such a use case arises, we can remove the assert and + // implement the logic to handle different sizes. + assert(storage_blob_size == query_blob_size); + + preprocess(original_blob, storage_blob, query_blob, storage_blob_size, alignment); + } void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const override { @@ -111,6 +121,17 @@ class DummyQueryPreprocessor : public PreprocessorInterface { value_to_add_query = value_to_add_storage; } + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + // This assert verifies that there's no use for this function for now - different sizes for + // storage and query blobs. If such a use case arises, we can remove the assert and + // implement the logic to handle different sizes. + assert(storage_blob_size == query_blob_size); + + preprocess(original_blob, storage_blob, query_blob, storage_blob_size, alignment); + } + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const override { this->preprocessQuery(original_blob, query_blob, input_blob_size, alignment); @@ -148,6 +169,12 @@ class DummyMixedPreprocessor : public PreprocessorInterface { DataType value_to_add_storage, DataType value_to_add_query) : PreprocessorInterface(allocator), value_to_add_storage(value_to_add_storage), value_to_add_query(value_to_add_query) {} + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + preprocess(original_blob, storage_blob, query_blob, storage_blob_size, alignment); + } + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const override { @@ -208,6 +235,18 @@ class DummyChangeAllocSizePreprocessor : public PreprocessorInterface { static constexpr unsigned char getExcessValue() { return excess_value; } + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + // if the blobs are equal, + if (storage_blob == query_blob) { + preprocessGeneral(original_blob, storage_blob, storage_blob_size, alignment); + query_blob = storage_blob; + query_blob_size = storage_blob_size; + } + } + + // If the input blob size is not enough void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const override { // if the blobs are equal, @@ -960,3 +999,296 @@ TEST(PreprocessorsTest, Int8NormalizeThenIncreaseSize) { final_blob_bytes_count)); } } + +// Tests the quantization preprocessor with a single preprocessor in the chain. +// The QuantPreprocessor allocates the storage blob and processes it, while the query blob +// is unprocessed and allocated by the preprocessors container. +TEST(PreprocessorsTest, QuantizationTest) { + std::shared_ptr allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t n_preprocessors = 1; + constexpr size_t alignment = 5; + constexpr size_t elements = 7; + constexpr size_t original_blob_size = elements * sizeof(float); + auto original_blob_alloc = allocator->allocate_unique(original_blob_size); + float *original_blob = static_cast(original_blob_alloc.get()); + test_utils::populate_float_vec(original_blob, elements); + + auto quant_preprocessor = new (allocator) QuantPreprocessor(allocator, elements); + auto multiPPContainer = + MultiPreprocessorsContainer(allocator, alignment); + multiPPContainer.addPreprocessor(quant_preprocessor); + + { + ProcessedBlobs processed_blobs = + multiPPContainer.preprocess(original_blob, original_blob_size); + const void *storage_blob = processed_blobs.getStorageBlob(); + const void *query_blob = processed_blobs.getQueryBlob(); + // blobs should point to the same memory slot + ASSERT_NE(storage_blob, nullptr); + ASSERT_NE(storage_blob, query_blob); + + constexpr size_t quantized_blob_bytes_count = + elements * sizeof(uint8_t) + 2 * sizeof(float); + auto expected_processed_blob_alloc = allocator->allocate_unique(quantized_blob_bytes_count); + uint8_t *expected_processed_blob = + static_cast(expected_processed_blob_alloc.get()); + + quant_preprocessor->quantize(original_blob, expected_processed_blob); + EXPECT_NO_FATAL_FAILURE(CompareVectors(static_cast(storage_blob), + expected_processed_blob, + quantized_blob_bytes_count)); + + // Compare the min and delta values of the quantized blob and expected_processed_blob + const float *storage_blob_metadata = + reinterpret_cast(static_cast(storage_blob) + elements); + const float *qexpected_processed_metadata = + reinterpret_cast(expected_processed_blob + elements); + // Check that the min and delta values are close enough + ASSERT_FLOAT_EQ(storage_blob_metadata[0], qexpected_processed_metadata[0]); + ASSERT_FLOAT_EQ(storage_blob_metadata[1], qexpected_processed_metadata[1]); + + float min = storage_blob_metadata[0]; + float delta = storage_blob_metadata[1]; + // reconstruct the original blob from the quantized blob + uint8_t *uint8_storage_blob = static_cast(const_cast(storage_blob)); + for (size_t i = 0; i < elements; i++) { + float reconstructed_value = min + uint8_storage_blob[i] * delta; + ASSERT_NEAR(reconstructed_value, original_blob[i], 0.01) + << "Reconstructed blob differs from the original blob at index " << i; + } + } +} + +// Tests the quantization preprocessor with a cosine preprocessor in the chain. +// The QuantPreprocessor receives an allocated blob from the cosine preprocessor. +TEST(PreprocessorsTest, QuantizationTestWithCosine) { + std::shared_ptr allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t n_preprocessors = 2; + constexpr size_t alignment = 5; + constexpr size_t elements = 7; + constexpr size_t original_blob_size = elements * sizeof(float); + auto original_blob_alloc = allocator->allocate_unique(original_blob_size); + float *original_blob = static_cast(original_blob_alloc.get()); + test_utils::populate_float_vec(original_blob, elements); + + auto quant_preprocessor = new (allocator) QuantPreprocessor(allocator, elements); + auto cosine_preprocessor = + new (allocator) CosinePreprocessor(allocator, elements, original_blob_size); + auto multiPPContainer = + MultiPreprocessorsContainer(allocator, alignment); + multiPPContainer.addPreprocessor(cosine_preprocessor); + multiPPContainer.addPreprocessor(quant_preprocessor); + + { + ProcessedBlobs processed_blobs = + multiPPContainer.preprocess(original_blob, original_blob_size); + const void *storage_blob = processed_blobs.getStorageBlob(); + const void *query_blob = processed_blobs.getQueryBlob(); + // blobs should point to the same memory slot + ASSERT_NE(storage_blob, nullptr); + ASSERT_NE(query_blob, nullptr); + ASSERT_NE(storage_blob, query_blob); + + auto expected_processed_blob_alloc = allocator->allocate_unique(original_blob_size); + float *expected_processed_blob = static_cast(expected_processed_blob_alloc.get()); + memcpy(expected_processed_blob, original_blob, original_blob_size); + VecSim_Normalize(expected_processed_blob, elements, VecSimType_FLOAT32); + + auto quantized_blob_alloc = + allocator->allocate_unique(elements * sizeof(uint8_t) + 2 * sizeof(float)); + uint8_t *quantized_blob = static_cast(quantized_blob_alloc.get()); + + // quantization should be applied after normalization + quant_preprocessor->quantize(expected_processed_blob, quantized_blob); + // compare the storage blob to the expected processed blob + EXPECT_NO_FATAL_FAILURE(CompareVectors(static_cast(storage_blob), + quantized_blob, elements)); + const float *storage_blob_metadata = + reinterpret_cast(static_cast(storage_blob) + elements); + const float *qexpected_processed_metadata = + reinterpret_cast(quantized_blob + elements); + // Check that the min and delta values are close enough + ASSERT_FLOAT_EQ(storage_blob_metadata[0], qexpected_processed_metadata[0]); + ASSERT_FLOAT_EQ(storage_blob_metadata[1], qexpected_processed_metadata[1]); + } +} + +// Tests the quantization preprocessor with a single preprocessor in the chain. +// The QuantPreprocessor allocates the storage blob with a size larger than the original blob. + +TEST(PreprocessorsTest, ReallocateVectorQuantizationTest) { + // Checks that if not enough memory was allocated by a previous preprocessor, the quantization + // preprocessor will reallocate it. + std::shared_ptr allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t n_preprocessors = 2; + constexpr size_t alignment = 5; + constexpr size_t elements = 2; + constexpr size_t original_blob_size = elements * sizeof(float); + auto original_blob_alloc = allocator->allocate_unique(original_blob_size); + float *original_blob = static_cast(original_blob_alloc.get()); + test_utils::populate_float_vec(original_blob, elements); + + auto quant_preprocessor = new (allocator) QuantPreprocessor(allocator, elements); + auto dummy_preprocessor = + new (allocator) dummyPreprocessors::DummyStoragePreprocessor(allocator, 0.0f); + auto multiPPContainer = + MultiPreprocessorsContainer(allocator, alignment); + multiPPContainer.addPreprocessor(dummy_preprocessor); + multiPPContainer.addPreprocessor(quant_preprocessor); + { + ProcessedBlobs processed_blobs = + multiPPContainer.preprocess(original_blob, original_blob_size); + const void *storage_blob = processed_blobs.getStorageBlob(); + const void *query_blob = processed_blobs.getQueryBlob(); + // blobs should point to the same memory slot + ASSERT_NE(storage_blob, nullptr); + ASSERT_NE(storage_blob, query_blob); + + constexpr size_t quantized_blob_bytes_count = + elements * sizeof(uint8_t) + 2 * sizeof(float); + uint8_t expected_processed_blob[quantized_blob_bytes_count] = {0}; + quant_preprocessor->quantize(original_blob, expected_processed_blob); + EXPECT_NO_FATAL_FAILURE(CompareVectors(static_cast(storage_blob), + expected_processed_blob, + quantized_blob_bytes_count)); + } +} + +// Tests the quantization preprocessor with a cosine preprocessor in the chain. +// The QuantPreprocessor receives an allocated blob from the cosine preprocessor, and needs to +// reallocate it. because the original blob size is smaller than the processed_bytes_count of the +// cosine preprocessor. +TEST(PreprocessorsTest, ReallocateVectorCosineQuantizationTest) { + // Checks that if not enough memory was allocated by a previous preprocessor, the quantization + // preprocessor will reallocate it. + std::shared_ptr allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t n_preprocessors = 2; + constexpr size_t alignment = 5; + constexpr size_t elements = 2; + constexpr size_t original_blob_size = elements * sizeof(float); + auto original_blob_alloc = allocator->allocate_unique(original_blob_size); + float *original_blob = static_cast(original_blob_alloc.get()); + for (size_t i = 0; i < elements; i++) { + original_blob[i] = static_cast(i + 2.5f); + } + + auto quant_preprocessor = new (allocator) QuantPreprocessor(allocator, elements); + auto cosine_preprocessor = + new (allocator) CosinePreprocessor(allocator, elements, original_blob_size); + auto multiPPContainer = + MultiPreprocessorsContainer(allocator, alignment); + multiPPContainer.addPreprocessor(cosine_preprocessor); + multiPPContainer.addPreprocessor(quant_preprocessor); + + { + ProcessedBlobs processed_blobs = + multiPPContainer.preprocess(original_blob, original_blob_size); + const void *storage_blob = processed_blobs.getStorageBlob(); + const void *query_blob = processed_blobs.getQueryBlob(); + // blobs should point to the same memory slot + ASSERT_NE(storage_blob, nullptr); + ASSERT_NE(query_blob, nullptr); + ASSERT_NE(storage_blob, query_blob); + + float expected_processed_blob[elements] = {0}; + memcpy(expected_processed_blob, original_blob, original_blob_size); + VecSim_Normalize(expected_processed_blob, elements, VecSimType_FLOAT32); + uint8_t quantized_blob[elements * sizeof(uint8_t) + 2 * sizeof(float)] = {0}; + // quantization should be applied after normalization + quant_preprocessor->quantize(expected_processed_blob, quantized_blob); + // compare the storage blob to the expected processed blob + EXPECT_NO_FATAL_FAILURE(CompareVectors(static_cast(storage_blob), + quantized_blob, elements)); + } +} + +TEST(PreprocessorsTest, QuantizationInPlaceTest) { + std::shared_ptr allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t alignment = 5; + constexpr size_t dim = 5; + constexpr size_t n_preprocessors = 2; + constexpr size_t original_blob_size = dim * sizeof(float); + constexpr size_t storage_bytes_count = dim * sizeof(uint8_t) + 2 * sizeof(float); + + // Create a float array with known values + float original_data[dim] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + + auto quant_preprocessor = new (allocator) QuantPreprocessor(allocator, dim); + auto dummy_preprocessor = + new (allocator) dummyPreprocessors::DummyStoragePreprocessor(allocator, 0.0f); + auto multiPPContainer = + MultiPreprocessorsContainer(allocator, alignment); + multiPPContainer.addPreprocessor(dummy_preprocessor); + multiPPContainer.addPreprocessor(quant_preprocessor); + { + ProcessedBlobs processed_blobs = + multiPPContainer.preprocess(original_data, original_blob_size); + const void *storage_blob = processed_blobs.getStorageBlob(); + const void *query_blob = processed_blobs.getQueryBlob(); + // blobs should point to the same memory slot + ASSERT_NE(storage_blob, nullptr); + ASSERT_NE(storage_blob, query_blob); + + // Verify the quantization results + const uint8_t *quantized = static_cast(storage_blob); + const float *metadata = reinterpret_cast(quantized + dim); + + // Extract metadata + float min_val = metadata[0]; + float delta = metadata[1]; + + // Check min_val is correct (should be 1.0f) + ASSERT_FLOAT_EQ(min_val, 1.0f); + + // Check delta is correct ((5.0f - 1.0f) / 255.0f) + ASSERT_FLOAT_EQ(delta, 4.0f / 255.0f); + + // dequantize and verify the values + for (size_t i = 0; i < dim; ++i) { + float dequantized_value = min_val + quantized[i] * delta; + ASSERT_NEAR(dequantized_value, original_data[i], 0.01); + } + } +} + +// Test the backward compatibility of the preprocess method with a single input_blob_size parameter +TEST(PreprocessorsTest, PreprocessBackwardCompatibilityTest) { + using namespace dummyPreprocessors; + std::shared_ptr allocator = VecSimAllocator::newVecsimAllocator(); + + constexpr size_t dim = 4; + unsigned char alignment = 5; + float initial_value = 1.0f; + const float original_blob[dim] = {initial_value, initial_value, initial_value, initial_value}; + size_t original_blob_size = sizeof(original_blob); + + // Create a preprocessor that modifies both storage and query blobs + auto mixed_preprocessor = new (allocator) DummyMixedPreprocessor(allocator, 7.0f, 2.0f); + + // Test the backward compatibility method (single input_blob_size) + void *storage_blob = nullptr; + void *query_blob = nullptr; + size_t input_blob_size = original_blob_size; + + // Call the backward compatibility version of preprocess + mixed_preprocessor->preprocess(original_blob, storage_blob, query_blob, input_blob_size, + alignment); + + // Verify that both blobs were allocated and processed correctly + ASSERT_NE(storage_blob, nullptr); + ASSERT_NE(query_blob, nullptr); + ASSERT_NE(storage_blob, query_blob); + + // Verify that the input_blob_size was updated correctly + ASSERT_EQ(input_blob_size, original_blob_size); + + // Verify that the storage blob was processed correctly + ASSERT_EQ(((const float *)storage_blob)[0], initial_value + 7.0f); + + // Verify that the query blob was processed correctly + ASSERT_EQ(((const float *)query_blob)[0], initial_value + 2.0f); + + // Verify that the query blob is aligned + unsigned char address_alignment = (uintptr_t)(query_blob) % alignment; + ASSERT_EQ(address_alignment, 0); +} diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 0b7157c0c..28a0bc1b2 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -4204,6 +4204,17 @@ class PreprocessorDoubleValue : public PreprocessorInterface { public: PreprocessorDoubleValue(std::shared_ptr allocator, size_t dim) : PreprocessorInterface(allocator), dim(dim) {} + + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char alignment) const override { + // This assert makes sure the current use of the preprocessor is valid, + // i.e., both blobs are of the same size. + // In order to use different sizes, the preprocessor should be modified. + assert(storage_blob_size == query_blob_size); + preprocess(original_blob, storage_blob, query_blob, storage_blob_size, alignment); + } + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, size_t &input_blob_size, unsigned char alignment) const override {