From ae27f6f8cd3090671d6e0badfac9d744156c49ed Mon Sep 17 00:00:00 2001 From: Narek Galstyan Date: Sun, 15 Oct 2023 22:26:26 +0000 Subject: [PATCH] Make sure memory-exceeded warnings are triggered in tests --- src/hnsw/external_index.c | 2 +- test/expected/hnsw_create.out | 8 ++++++++ test/expected/hnsw_insert.out | 10 +++++++++- test/sql/hnsw_create.sql | 12 ++++++++++++ test/sql/hnsw_insert.sql | 11 ++++++++++- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/hnsw/external_index.c b/src/hnsw/external_index.c index aee3aa9de..ae3bbbd4a 100644 --- a/src/hnsw/external_index.c +++ b/src/hnsw/external_index.c @@ -638,7 +638,7 @@ void *ldb_wal_index_node_retriever(void *ctxp, int id) NULL, NULL, 0, - "Pinned more tuples during node retrieval than will fir in work_mem, cosider increasing work_mem"); + "pinned more tuples during node retrieval than will fit in work_mem, cosider increasing work_mem"); #endif fa_cache_insert(&ctx->fa_cache, id, nodepage->node); diff --git a/test/expected/hnsw_create.out b/test/expected/hnsw_create.out index 28eb6a63a..77a77599b 100644 --- a/test/expected/hnsw_create.out +++ b/test/expected/hnsw_create.out @@ -23,11 +23,19 @@ CREATE TABLE IF NOT EXISTS sift_base1k ( v REAL[] ); COPY sift_base1k (v) FROM '/tmp/lantern/vector_datasets/sift_base1k_arrays.csv' WITH csv; +-- limit work_mem to trigger codepaths for exceeding work_mem +set work_mem = '64kB'; +set client_min_messages = 'ERROR'; -- Validate that creating a secondary index works CREATE INDEX ON sift_base1k USING hnsw (v) WITH (dim=128, M=4); INFO: done init usearch index INFO: inserted 1000 elements INFO: done saving 1000 vectors +-- We do not actually print the warnings generated for exceeding work_mem because the work_mem +-- check does not work for postgres 13 and lower.So, if we printed the warnings, we would get a regression +-- failure in older postgres versions. We still reduce workmem to exercise relevant codepaths for coverage +set client_min_messages = 'WARNING'; +set work_mem = '10MB'; SELECT * FROM ldb_get_indexes('sift_base1k'); indexname | size | indexdef | total_index_size -------------------+--------+---------------------------------------------------------------------------------------------+------------------ diff --git a/test/expected/hnsw_insert.out b/test/expected/hnsw_insert.out index d99dc75b6..6140c130b 100644 --- a/test/expected/hnsw_insert.out +++ b/test/expected/hnsw_insert.out @@ -1,7 +1,12 @@ --------------------------------------------------------------------- -- Test HNSW index inserts on empty table --------------------------------------------------------------------- -set work_mem = '10MB'; +-- set an artificially low work_mem to make sure work_mem exceeded warnings are printed +set work_mem = '64kB'; +-- We do not actually print the warnings generated for exceeding work_mem because the work_mem +-- check does not work for postgres 13 and lower.So, if we printed the warnings, we would get a regression +-- failure in older postgres versions. We still reduce workmem to exercise relevant codepaths for coverage +set client_min_messages = 'ERROR'; CREATE TABLE small_world ( id SERIAL PRIMARY KEY, v REAL[2] @@ -19,6 +24,9 @@ INSERT INTO small_world (v) VALUES ('{1,1,1,1}'); ERROR: Wrong number of dimensions: 4 instead of 3 expected \set ON_ERROR_STOP on DROP TABLE small_world; +-- set work_mem to a value that is enough for the tests +set client_min_messages = 'WARNING'; +set work_mem = '10MB'; --------------------------------------------------------------------- -- Test HNSW index inserts on non-empty table --------------------------------------------------------------------- diff --git a/test/sql/hnsw_create.sql b/test/sql/hnsw_create.sql index 7845575c9..70d72b9f5 100644 --- a/test/sql/hnsw_create.sql +++ b/test/sql/hnsw_create.sql @@ -6,8 +6,19 @@ \ir utils/small_world_array.sql \ir utils/sift1k_array.sql +-- limit work_mem to trigger codepaths for exceeding work_mem +set work_mem = '64kB'; +set client_min_messages = 'ERROR'; + -- Validate that creating a secondary index works CREATE INDEX ON sift_base1k USING hnsw (v) WITH (dim=128, M=4); +-- We do not actually print the warnings generated for exceeding work_mem because the work_mem +-- check does not work for postgres 13 and lower.So, if we printed the warnings, we would get a regression +-- failure in older postgres versions. We still reduce workmem to exercise relevant codepaths for coverage + +set client_min_messages = 'WARNING'; +set work_mem = '10MB'; + SELECT * FROM ldb_get_indexes('sift_base1k'); -- Validate that index creation works with a larger number of vectors @@ -17,6 +28,7 @@ SELECT v AS v4444 FROM sift_base10k WHERE id = 4444 \gset EXPLAIN (COSTS FALSE) SELECT * FROM sift_base10k order by v <-> :'v4444' LIMIT 10; --- Validate that M values inside the allowed range [2, 128] do not throw an error + CREATE INDEX ON small_world USING hnsw (v) WITH (M=2); CREATE INDEX ON small_world USING hnsw (v) WITH (M=128); diff --git a/test/sql/hnsw_insert.sql b/test/sql/hnsw_insert.sql index d55e6184e..39e72187b 100644 --- a/test/sql/hnsw_insert.sql +++ b/test/sql/hnsw_insert.sql @@ -1,7 +1,12 @@ --------------------------------------------------------------------- -- Test HNSW index inserts on empty table --------------------------------------------------------------------- -set work_mem = '10MB'; +-- set an artificially low work_mem to make sure work_mem exceeded warnings are printed +set work_mem = '64kB'; +-- We do not actually print the warnings generated for exceeding work_mem because the work_mem +-- check does not work for postgres 13 and lower.So, if we printed the warnings, we would get a regression +-- failure in older postgres versions. We still reduce workmem to exercise relevant codepaths for coverage +set client_min_messages = 'ERROR'; CREATE TABLE small_world ( id SERIAL PRIMARY KEY, @@ -20,6 +25,10 @@ INSERT INTO small_world (v) VALUES ('{1,1,1,1}'); DROP TABLE small_world; +-- set work_mem to a value that is enough for the tests +set client_min_messages = 'WARNING'; +set work_mem = '10MB'; + --------------------------------------------------------------------- -- Test HNSW index inserts on non-empty table ---------------------------------------------------------------------