Skip to content

Commit

Permalink
add test back in to external_index, fix elog in CheckMem
Browse files Browse the repository at this point in the history
  • Loading branch information
ezra-varady committed Oct 14, 2023
1 parent e85e05b commit 1175c02
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 8 deletions.
7 changes: 7 additions & 0 deletions src/hnsw/external_index.c
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <access/generic_xlog.h> // GenericXLog
#include <assert.h>
#include <common/relpath.h>
#include <miscadmin.h>
#include <pg_config.h> // BLCKSZ
#include <storage/bufmgr.h> // Buffer
#include <utils/hsearch.h>
Expand Down Expand Up @@ -629,6 +630,12 @@ void *ldb_wal_index_node_retriever(void *ctxp, int id)
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
}

CheckMem(work_mem,
NULL,
NULL,
0,
"Pinned more tuples during node retrieval than will fir in work_mem, cosider increasing work_mem");

cache_set_item(&ctx->node_cache, &id, nodepage->node);

return nodepage->node;
Expand Down
19 changes: 11 additions & 8 deletions src/hnsw/utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -55,18 +55,21 @@ usearch_label_t GetUsearchLabel(ItemPointer itemPtr)

void CheckMem(int limit, Relation index, usearch_index_t uidx, uint32 n_nodes, char *msg)
{
usearch_error_t error;
double M = ldb_HnswGetM(index);
double mL = 1 / log(M);
usearch_metadata_t meta = usearch_metadata(uidx, &error);
// todo:: update sizeof(float) to correct vector size once #19 is merged
uint32 node_size = UsearchNodeBytes(&meta, meta.dimensions * sizeof(float), (int)round(mL + 1));
Size pg_mem = MemoryContextMemAllocated(CurrentMemoryContext, true);
uint32 node_size = 0;
if(index != NULL) {
usearch_error_t error;
double M = ldb_HnswGetM(index);
double mL = 1 / log(M);
usearch_metadata_t meta = usearch_metadata(uidx, &error);
// todo:: update sizeof(float) to correct vector size once #19 is merged
node_size = UsearchNodeBytes(&meta, meta.dimensions * sizeof(float), (int)round(mL + 1));
}
Size pg_mem = MemoryContextMemAllocated(CurrentMemoryContext, true);

// The average number of layers for an element to be added in is mL+1 per section 4.2.2
// Accuracy could maybe be improved by not rounding
// This is a guess, but it's a reasonably good one
if(pg_mem + node_size * n_nodes > (uint32)limit * 1024UL) {
elog(WARNING, msg);
elog(WARNING, "%s", msg);
}
}

0 comments on commit 1175c02

Please sign in to comment.