make exceeding work_mem a warning, change tests so they dont trigger it

lanterndata · Oct 11, 2023 · b1f1c29 · b1f1c29
1 parent 479a69b
commit b1f1c29
Show file tree

Hide file tree

Showing 9 changed files with 10 additions and 7 deletions.
diff --git a/src/hnsw/external_index.c b/src/hnsw/external_index.c
@@ -638,7 +638,7 @@ void *ldb_wal_index_node_retriever(void *ctxp, int id)
 #endif
         }
         if(ctx->memory >= work_mem * 1024L) {
-            elog(ERROR, "pinned more buffers during query than will fit in work_mem, consider increasing work_mem");
+            elog(WARNING, "pinned more buffers during query than will fit in work_mem, consider increasing work_mem");
         }
     }
     if(!idx_page_prelocked) {

diff --git a/src/hnsw/insert.c b/src/hnsw/insert.c
@@ -150,8 +150,7 @@ bool ldb_aminsert(Relation         index,
     uint32 node_size = UsearchNodeBytes(&meta, opts.dimensions * sizeof(float), (int)(mL + .5));
     // accuracy could be improved by not rounding mL, but otherwise this will never be fully accurate
     if (node_size * (hdr->num_vectors + 1) > work_mem * 1024L) {
-        usearch_free(uidx, &error);
-        elog(ERROR, "index size exceeded work_mem during insert");
+        elog(WARNING, "index size exceeded work_mem during insert");
     }
 
     usearch_reserve(uidx, hdr->num_vectors + 1, &error);

diff --git a/src/hnsw/scan.c b/src/hnsw/scan.c
@@ -202,8 +202,7 @@ bool ldb_amgettuple(IndexScanDesc scan, ScanDirection dir)
         // I think because of mem_view_lazy a max of k nodes will be held in memory by usearch
         // there are separate checks on the memory held by takenbuffers
         if (node_size * k > work_mem * 1024L) {
-            usearch_free(scanstate->usearch_index, &error);
-            elog(ERROR, "index size exceeded work_mem during insert");
+            elog(WARNING, "index size exceeded work_mem during scan");
         }
 
         ldb_dlog("LANTERN querying index for %d elements", k);
@@ -246,8 +245,7 @@ bool ldb_amgettuple(IndexScanDesc scan, ScanDirection dir)
         usearch_metadata_t meta = usearch_metadata(scanstate->usearch_index, &error);
         uint32 node_size = UsearchNodeBytes(&meta, scanstate->dimensions * sizeof(float), (int)(mL + .5));
         if (node_size * k > work_mem * 1024L) {
-            usearch_free(scanstate->usearch_index, &error);
-            elog(ERROR, "index size exceeded work_mem during insert");
+            elog(WARNING, "index size exceeded work_mem during scan");
         }
 
         ldb_dlog("LANTERN - querying index for %d elements", k);

diff --git a/test/parallel/expected/insert.out b/test/parallel/expected/insert.out
@@ -1,3 +1,4 @@
+SET work_mem='10MB';
 BEGIN;
 INSERT INTO sift_base10k (id, v) VALUES 
     (nextval('serial'), random_array(128, 0, 128)),

diff --git a/test/parallel/expected/insert2.out b/test/parallel/expected/insert2.out
@@ -1,3 +1,4 @@
+SET work_mem='10MB';
 BEGIN;
 INSERT INTO sift_base10k (id, v) VALUES 
     (nextval('serial'), random_array(128, 0, 128)),

diff --git a/test/parallel/expected/insert3.out b/test/parallel/expected/insert3.out
@@ -1,3 +1,4 @@
+SET work_mem='10MB';
 BEGIN;
 INSERT INTO sift_base10k (id, v) VALUES 
     (nextval('serial'), random_array(128, 0, 128)),

diff --git a/test/parallel/sql/insert.sql b/test/parallel/sql/insert.sql
@@ -1,3 +1,4 @@
+SET work_mem='10MB';
 BEGIN;
 INSERT INTO sift_base10k (id, v) VALUES 
     (nextval('serial'), random_array(128, 0, 128)),

diff --git a/test/parallel/sql/insert2.sql b/test/parallel/sql/insert2.sql
@@ -1,3 +1,4 @@
+SET work_mem='10MB';
 BEGIN;
 INSERT INTO sift_base10k (id, v) VALUES 
     (nextval('serial'), random_array(128, 0, 128)),

diff --git a/test/parallel/sql/insert3.sql b/test/parallel/sql/insert3.sql
@@ -1,3 +1,4 @@
+SET work_mem='10MB';
 BEGIN;
 INSERT INTO sift_base10k (id, v) VALUES 
     (nextval('serial'), random_array(128, 0, 128)),