initial implementation of parallel test running

CMakeLists.txt

@@ -192,7 +192,7 @@ add_custom_target(
 # TEST
 add_custom_target(
   test
-  COMMAND ${CMAKE_SOURCE_DIR}/scripts/run_all_tests.sh --regression
+  COMMAND ${CMAKE_SOURCE_DIR}/scripts/run_all_tests.sh --regression && ${CMAKE_SOURCE_DIR}/scripts/run_all_tests.sh --parallel
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/test
 )
 

CONTRIBUTING.md

@@ -8,7 +8,11 @@ make test
 
 # only run regression tests that have $FILTER in regression sql file path
 make test FILTER=hnsw
+
+# run parallel tests
+make test-parallel
 ```
+Running `make test` will run the lantern regression tests, these run independent of one another. At the moment the tests for `make test-parallel` are under development, they can be found in `test/parallel`. The goal of the parallel tests is to generate a more realistic workload on the index to discover timing errors and other bugs dependent on more complex use, they run in the same database. 
 
 ## Running benchmarks
 This requires Python to be installed. Please check the `Dockerfile.dev` for pip requirements.
@@ -30,7 +34,7 @@ If you build Lantern in a different directory, make sure to update `.vscode` con
 
 ## Debugging the C codebase
 
-If you make changes to the C codebase, in addition to `make test`, you can also use the `livedebug.py` utility in a `tmux` session to easily attach `gdb` to the psql backend and find out what breaks.
+If you make changes to the C codebase, in addition to `make test` and `make parallel-test`, you can also use the `livedebug.py` utility in a `tmux` session to easily attach `gdb` to the psql backend and find out what breaks.
 Below is a short recording demonstrating the use of `livedebug.py`:
 
 [![asciicast](https://asciinema.org/a/jTsbWdOcTvUl4iAJlAw3Cszbt.svg)](https://asciinema.org/a/jTsbWdOcTvUl4iAJlAw3Cszbt)

test/parallel/expected/begin.out

@@ -1,9 +1,10 @@
-CREATE TABLE small_world (
-    id varchar(3),
-    b boolean,
-    v real[3]
+\ir utils/sift10k_array.sql
+CREATE TABLE IF NOT EXISTS sift_base10k (
+     id SERIAL PRIMARY KEY,
+     v REAL[128]
 );
-CREATE INDEX ON small_world USING HNSW (v) WITH (dim=3, M=5, ef=20, ef_construction=20);
+\copy sift_base10k (v) FROM '/tmp/lantern/vector_datasets/siftsmall_base_arrays.csv' with csv;
+CREATE INDEX ON sift_base10k  USING HNSW (v) WITH (M=5, ef=20, ef_construction=20);
 INFO:  done init usearch index
-INFO:  inserted 0 elements
-INFO:  done saving 0 vectors
+INFO:  inserted 10000 elements
+INFO:  done saving 10000 vectors

test/parallel/expected/end.out

@@ -1,7 +1,12 @@
-BING BING WAHOO
-SELECT COUNT(*) FROM small_world;
+SELECT COUNT(*) FROM sift_base10k;
  count 
 -------
-    16
+ 10030
+(1 row)
+
+SELECT * from sift_base10k WHERE id=4444;
+  id  |                                                                                                                                                                             v                                                                                                                                                                             
+------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ 4444 | {55,61,11,4,5,2,13,24,65,49,13,9,23,37,94,38,54,11,14,14,40,31,50,44,53,4,0,0,27,17,8,34,12,10,4,4,22,52,68,53,9,2,0,0,2,116,119,64,119,2,0,0,2,30,119,119,116,5,0,8,47,9,5,60,7,7,10,23,56,50,23,5,28,68,6,18,24,65,50,9,119,75,3,0,1,8,12,85,119,11,4,6,8,9,5,74,25,11,8,20,18,12,2,21,11,90,25,32,33,15,2,9,84,67,8,4,22,31,11,33,119,30,3,6,0,0,0,26}
 (1 row)
 

test/parallel/expected/insert.out

@@ -1,9 +1,17 @@
-insert into small_world (id, b, v) values
-    ('000', true,  '{0,0,0}'),
-    ('001', true,  '{0,0,1}'),
-    ('010', false, '{0,1,0}'),
-    ('011', true,  '{0,1,1}'),
-    ('100', false, '{1,0,0}'),
-    ('101', false, '{1,0,1}'),
-    ('110', false, '{1,1,0}'),
-    ('111', true,  '{1,1,1}');
+\ir utils/random_array.sql
+CREATE OR REPLACE FUNCTION random_int_array(dim integer, min integer, max integer) RETURNS integer[] AS $BODY$
+begin
+        return (select array_agg(round(random() * (max - min)) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION random_array(dim integer, min real, max real) RETURNS REAL[] AS $BODY$
+begin
+        return (select array_agg(random() * (max - min) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+DO $$
+BEGIN
+    FOR i IN 1..10 LOOP
+        INSERT INTO sift_base10k (v) VALUES (random_array(128, 0, 128));
+    END LOOP;
+END; $$

test/parallel/expected/insert2.out

@@ -1,9 +1,17 @@
-insert into small_world (id, b, v) values
-    ('000', true,  '{0,0,0}'),
-    ('001', true,  '{0,0,1}'),
-    ('010', false, '{0,1,0}'),
-    ('011', true,  '{0,1,1}'),
-    ('100', false, '{1,0,0}'),
-    ('101', false, '{1,0,1}'),
-    ('110', false, '{1,1,0}'),
-    ('111', true,  '{1,1,1}');
+\ir utils/random_array.sql
+CREATE OR REPLACE FUNCTION random_int_array(dim integer, min integer, max integer) RETURNS integer[] AS $BODY$
+begin
+        return (select array_agg(round(random() * (max - min)) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION random_array(dim integer, min real, max real) RETURNS REAL[] AS $BODY$
+begin
+        return (select array_agg(random() * (max - min) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+DO $$
+BEGIN
+    FOR i IN 1..10 LOOP
+        INSERT INTO sift_base10k (v) VALUES (random_array(128, 0, 128));
+    END LOOP;
+END; $$

test/parallel/expected/insert3.out

@@ -0,0 +1,17 @@
+\ir utils/random_array.sql
+CREATE OR REPLACE FUNCTION random_int_array(dim integer, min integer, max integer) RETURNS integer[] AS $BODY$
+begin
+        return (select array_agg(round(random() * (max - min)) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION random_array(dim integer, min real, max real) RETURNS REAL[] AS $BODY$
+begin
+        return (select array_agg(random() * (max - min) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+DO $$
+BEGIN
+    FOR i IN 1..10 LOOP
+        INSERT INTO sift_base10k (v) VALUES (random_array(128, 0, 128));
+    END LOOP;
+END; $$

test/parallel/parallel_test_runner.sh

@@ -21,7 +21,6 @@ EOF
 }
 
 if [[ "$TESTFILE_NAME" =~ ^end ]]; then
-    echo "BING BING WAHOO";
     trap drop_db EXIT
 fi
 

test/parallel/sql/begin.sql

@@ -1,7 +1,3 @@
-CREATE TABLE small_world (
-    id varchar(3),
-    b boolean,
-    v real[3]
-);
+\ir utils/sift10k_array.sql
 
-CREATE INDEX ON small_world USING HNSW (v) WITH (dim=3, M=5, ef=20, ef_construction=20);
+CREATE INDEX ON sift_base10k  USING HNSW (v) WITH (M=5, ef=20, ef_construction=20);

test/parallel/sql/end.sql

@@ -1 +1,2 @@
-SELECT COUNT(*) FROM small_world;
+SELECT COUNT(*) FROM sift_base10k;
+SELECT * from sift_base10k WHERE id=4444;

test/parallel/sql/insert.sql

@@ -1,9 +1,7 @@
-insert into small_world (id, b, v) values
-    ('000', true,  '{0,0,0}'),
-    ('001', true,  '{0,0,1}'),
-    ('010', false, '{0,1,0}'),
-    ('011', true,  '{0,1,1}'),
-    ('100', false, '{1,0,0}'),
-    ('101', false, '{1,0,1}'),
-    ('110', false, '{1,1,0}'),
-    ('111', true,  '{1,1,1}');
+\ir utils/random_array.sql
+DO $$
+BEGIN
+    FOR i IN 1..10 LOOP
+        INSERT INTO sift_base10k (v) VALUES (random_array(128, 0, 128));
+    END LOOP;
+END; $$

test/parallel/sql/insert2.sql

@@ -1,9 +1,7 @@
-insert into small_world (id, b, v) values
-    ('000', true,  '{0,0,0}'),
-    ('001', true,  '{0,0,1}'),
-    ('010', false, '{0,1,0}'),
-    ('011', true,  '{0,1,1}'),
-    ('100', false, '{1,0,0}'),
-    ('101', false, '{1,0,1}'),
-    ('110', false, '{1,1,0}'),
-    ('111', true,  '{1,1,1}');
+\ir utils/random_array.sql
+DO $$
+BEGIN
+    FOR i IN 1..10 LOOP
+        INSERT INTO sift_base10k (v) VALUES (random_array(128, 0, 128));
+    END LOOP;
+END; $$

test/parallel/sql/insert3.sql

@@ -0,0 +1,7 @@
+\ir utils/random_array.sql
+DO $$
+BEGIN
+    FOR i IN 1..10 LOOP
+        INSERT INTO sift_base10k (v) VALUES (random_array(128, 0, 128));
+    END LOOP;
+END; $$

test/parallel/sql/utils/random_array.sql

@@ -0,0 +1,11 @@
+CREATE OR REPLACE FUNCTION random_int_array(dim integer, min integer, max integer) RETURNS integer[] AS $BODY$
+begin
+        return (select array_agg(round(random() * (max - min)) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION random_array(dim integer, min real, max real) RETURNS REAL[] AS $BODY$
+begin
+        return (select array_agg(random() * (max - min) + min) from generate_series (0, dim - 1));
+end
+$BODY$ LANGUAGE plpgsql;

test/parallel/sql/utils/sift10k_array.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS sift_base10k (
+     id SERIAL PRIMARY KEY,
+     v REAL[128]
+);
+\copy sift_base10k (v) FROM '/tmp/lantern/vector_datasets/siftsmall_base_arrays.csv' with csv;

test/parallel_schedule.txt

@@ -4,5 +4,5 @@
 # - 'test' lines may have multiple space-separated tests. All tests in a single 'test' line will be run in parallel
 
 test_begin: begin
-test: insert insert2
+test: insert insert2 insert3
 test_end: end

test/sql/utils/small_world_array.sql

@@ -1,15 +1,15 @@
-create table small_world (
-    id varchar(3),
-    b boolean,
-    v real[3]
+CREATE TABLE small_world (
+    id VARCHAR(3),
+    b BOOLEAN,
+    v REAL[3]
 );
 
-insert into small_world (id, b, v) values
-    ('000', true,  '{0,0,0}'),
-    ('001', true,  '{0,0,1}'),
-    ('010', false, '{0,1,0}'),
-    ('011', true,  '{0,1,1}'),
-    ('100', false, '{1,0,0}'),
-    ('101', false, '{1,0,1}'),
-    ('110', false, '{1,1,0}'),
-    ('111', true,  '{1,1,1}');
+INSERT INTO small_world (id, b, v) VALUES
+    ('000', TRUE,  '{0,0,0}'),
+    ('001', TRUE,  '{0,0,1}'),
+    ('010', FALSE, '{0,1,0}'),
+    ('011', TRUE,  '{0,1,1}'),
+    ('100', FALSE, '{1,0,0}'),
+    ('101', FALSE, '{1,0,1}'),
+    ('110', FALSE, '{1,1,0}'),
+    ('111', TRUE,  '{1,1,1}');