Skip to content

Commit ea7a97a

Browse files
tobyhedetobyhede
authored
Merge pull request #101 from cipherstash/eql-v2-0-jsonb
Eql v2.0 - Searchable Encrypted JSONB
2 parents 71c86bb + 91a9c92 commit ea7a97a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+4298
-1753
lines changed

‎diagrams/overview-insert.drawio.svg

-457
This file was deleted.

‎diagrams/overview-select.drawio.svg

-552
This file was deleted.

‎src/blake3/functions.sql

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
-- REQUIRE: src/schema.sql
2+
3+
-- extracts ste_vec index from a jsonb value
4+
-- DROP FUNCTION IF EXISTS eql_v1.blake3(val jsonb);
5+
6+
-- extracts blake3 index from a jsonb value
7+
-- DROP FUNCTION IF EXISTS eql_v1.blake3(val jsonb);
8+
9+
CREATE FUNCTION eql_v1.blake3(val jsonb)
10+
RETURNS eql_v1.blake3
11+
IMMUTABLE STRICT PARALLEL SAFE
12+
AS $$
13+
BEGIN
14+
15+
IF NOT (val ? 'b') NULL THEN
16+
RAISE 'Expected a blake3 index (b) value in json: %', val;
17+
END IF;
18+
19+
IF val->>'b' IS NULL THEN
20+
RETURN NULL;
21+
END IF;
22+
23+
RETURN val->>'b';
24+
END;
25+
$$ LANGUAGE plpgsql;
26+
27+
28+
-- extracts blake3 index from an eql_v1_encrypted value
29+
-- DROP FUNCTION IF EXISTS eql_v1.blake3(val eql_v1_encrypted);
30+
31+
CREATE FUNCTION eql_v1.blake3(val eql_v1_encrypted)
32+
RETURNS eql_v1.blake3
33+
IMMUTABLE STRICT PARALLEL SAFE
34+
AS $$
35+
BEGIN
36+
RETURN (SELECT eql_v1.blake3(val.data));
37+
END;
38+
$$ LANGUAGE plpgsql;

‎src/blake3/types.sql

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- REQUIRE: src/schema.sql
2+
3+
-- DROP DOMAIN IF EXISTS eql_v1.blake3;
4+
CREATE DOMAIN eql_v1.blake3 AS text;

‎src/common.sql

+56-2
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,65 @@
22
-- REQUIRE: src/schema.sql
33

44

5+
-- Constant time comparison of 2 bytea values
6+
7+
8+
9+
10+
11+
12+
-- DROP FUNCTION IF EXISTS eql_v1.bytea_eq(a bytea, b bytea);
13+
14+
CREATE FUNCTION eql_v1.bytea_eq(a bytea, b bytea) RETURNS boolean AS $$
15+
DECLARE
16+
result boolean;
17+
differing bytea;
18+
BEGIN
19+
20+
-- Check if the bytea values are the same length
21+
IF LENGTH(a) != LENGTH(b) THEN
22+
RETURN false;
23+
END IF;
24+
25+
-- Compare each byte in the bytea values
26+
result := true;
27+
FOR i IN 1..LENGTH(a) LOOP
28+
IF SUBSTRING(a FROM i FOR 1) != SUBSTRING(b FROM i FOR 1) THEN
29+
result := result AND false;
30+
END IF;
31+
END LOOP;
32+
33+
RETURN result;
34+
END;
35+
$$ LANGUAGE plpgsql;
36+
37+
38+
-- DROP FUNCTION IF EXISTS eql_v1.jsonb_array_to_bytea_array(val jsonb);
39+
40+
-- Casts a jsonb array of hex-encoded strings to an array of bytea.
41+
CREATE FUNCTION eql_v1.jsonb_array_to_bytea_array(val jsonb)
42+
RETURNS bytea[] AS $$
43+
DECLARE
44+
terms_arr bytea[];
45+
BEGIN
46+
IF jsonb_typeof(val) = 'null' THEN
47+
RETURN NULL;
48+
END IF;
49+
50+
SELECT array_agg(decode(value::text, 'hex')::bytea)
51+
INTO terms_arr
52+
FROM jsonb_array_elements_text(val) AS value;
53+
54+
RETURN terms_arr;
55+
END;
56+
$$ LANGUAGE plpgsql;
57+
58+
559

660
--
761
-- Convenience function to log a message
862
--
9-
DROP FUNCTION IF EXISTS eql_v1.log(text);
63+
-- DROP FUNCTION IF EXISTS eql_v1.log(text);
1064
CREATE FUNCTION eql_v1.log(s text)
1165
RETURNS void
1266
AS $$
@@ -19,7 +73,7 @@ $$ LANGUAGE plpgsql;
1973
--
2074
-- Convenience function to describe a test
2175
--
22-
DROP FUNCTION IF EXISTS eql_v1.log(text, text);
76+
-- DROP FUNCTION IF EXISTS eql_v1.log(text, text);
2377
CREATE FUNCTION eql_v1.log(ctx text, s text)
2478
RETURNS void
2579
AS $$

‎src/config/config_test.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
--
55
-- Helper function for assertions
66
--
7-
DROP FUNCTION IF EXISTS _index_exists(text, text, text, text);
7+
-- DROP FUNCTION IF EXISTS _index_exists(text, text, text, text);
88
CREATE FUNCTION _index_exists(table_name text, column_name text, index_name text, state text DEFAULT 'pending')
99
RETURNS boolean
1010
LANGUAGE sql STRICT PARALLEL SAFE

‎src/config/constraints.sql

+5-5
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
--
66
-- Used by the eql_v1.config_check_indexes as part of the configuration_data_v1 constraint
77
--
8-
DROP FUNCTION IF EXISTS eql_v1.config_get_indexes(jsonb);
8+
-- DROP FUNCTION IF EXISTS eql_v1.config_get_indexes(jsonb);
99
CREATE FUNCTION eql_v1.config_get_indexes(val jsonb)
1010
RETURNS SETOF text
1111
LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
@@ -18,7 +18,7 @@ END;
1818
--
1919
-- Used by the cs_configuration_data_v1_check constraint
2020
--
21-
DROP FUNCTION IF EXISTS eql_v1.config_check_indexes(jsonb);
21+
-- DROP FUNCTION IF EXISTS eql_v1.config_check_indexes(jsonb);
2222
CREATE FUNCTION eql_v1.config_check_indexes(val jsonb)
2323
RETURNS BOOLEAN
2424
IMMUTABLE STRICT PARALLEL SAFE
@@ -36,7 +36,7 @@ AS $$
3636
$$ LANGUAGE plpgsql;
3737

3838

39-
DROP FUNCTION IF EXISTS eql_v1.config_check_cast(jsonb);
39+
-- DROP FUNCTION IF EXISTS eql_v1.config_check_cast(jsonb);
4040

4141
CREATE FUNCTION eql_v1.config_check_cast(val jsonb)
4242
RETURNS BOOLEAN
@@ -52,7 +52,7 @@ $$ LANGUAGE plpgsql;
5252
--
5353
-- Should include a tables field
5454
-- Tables should not be empty
55-
DROP FUNCTION IF EXISTS eql_v1.config_check_tables(jsonb);
55+
-- DROP FUNCTION IF EXISTS eql_v1.config_check_tables(jsonb);
5656
CREATE FUNCTION eql_v1.config_check_tables(val jsonb)
5757
RETURNS boolean
5858
AS $$
@@ -65,7 +65,7 @@ AS $$
6565
$$ LANGUAGE plpgsql;
6666

6767
-- Should include a version field
68-
DROP FUNCTION IF EXISTS eql_v1.config_check_version(jsonb);
68+
-- DROP FUNCTION IF EXISTS eql_v1.config_check_version(jsonb);
6969
CREATE FUNCTION eql_v1.config_check_version(val jsonb)
7070
RETURNS boolean
7171
AS $$

‎src/config/functions.sql

+16-16
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
--
55
--
66

7-
DROP FUNCTION IF EXISTS eql_v1.config_default(config jsonb);
7+
-- DROP FUNCTION IF EXISTS eql_v1.config_default(config jsonb);
88

99
CREATE FUNCTION eql_v1.config_default(config jsonb)
1010
RETURNS jsonb
@@ -19,7 +19,7 @@ AS $$
1919
$$ LANGUAGE plpgsql;
2020

2121

22-
DROP FUNCTION IF EXISTS eql_v1.config_add_table(table_name text, config jsonb);
22+
-- DROP FUNCTION IF EXISTS eql_v1.config_add_table(table_name text, config jsonb);
2323

2424
CREATE FUNCTION eql_v1.config_add_table(table_name text, config jsonb)
2525
RETURNS jsonb
@@ -37,7 +37,7 @@ $$ LANGUAGE plpgsql;
3737

3838

3939
-- Add the column if it doesn't exist
40-
DROP FUNCTION IF EXISTS eql_v1.config_add_column(table_name text, column_name text, config jsonb);
40+
-- DROP FUNCTION IF EXISTS eql_v1.config_add_column(table_name text, column_name text, config jsonb);
4141

4242
CREATE FUNCTION eql_v1.config_add_column(table_name text, column_name text, config jsonb)
4343
RETURNS jsonb
@@ -56,7 +56,7 @@ $$ LANGUAGE plpgsql;
5656

5757

5858
-- Set the cast
59-
DROP FUNCTION IF EXISTS eql_v1.config_add_cast(table_name text, column_name text, cast_as text, config jsonb);
59+
-- DROP FUNCTION IF EXISTS eql_v1.config_add_cast(table_name text, column_name text, cast_as text, config jsonb);
6060

6161
CREATE FUNCTION eql_v1.config_add_cast(table_name text, column_name text, cast_as text, config jsonb)
6262
RETURNS jsonb
@@ -70,7 +70,7 @@ $$ LANGUAGE plpgsql;
7070

7171

7272
-- Add the column if it doesn't exist
73-
DROP FUNCTION IF EXISTS eql_v1.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb);
73+
-- DROP FUNCTION IF EXISTS eql_v1.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb);
7474

7575
CREATE FUNCTION eql_v1.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb)
7676
RETURNS jsonb
@@ -86,7 +86,7 @@ $$ LANGUAGE plpgsql;
8686
--
8787
-- Default options for match index
8888
--
89-
DROP FUNCTION IF EXISTS eql_v1.config_match_default();
89+
-- DROP FUNCTION IF EXISTS eql_v1.config_match_default();
9090

9191
CREATE FUNCTION eql_v1.config_match_default()
9292
RETURNS jsonb
@@ -103,7 +103,7 @@ END;
103103
--
104104
-- Adds an index term to the configuration
105105
--
106-
DROP FUNCTION IF EXISTS eql_v1.add_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
106+
-- DROP FUNCTION IF EXISTS eql_v1.add_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
107107

108108
CREATE FUNCTION eql_v1.add_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
109109
RETURNS jsonb
@@ -155,7 +155,7 @@ AS $$
155155
$$ LANGUAGE plpgsql;
156156

157157

158-
DROP FUNCTION IF EXISTS eql_v1.remove_index(table_name text, column_name text, index_name text);
158+
-- DROP FUNCTION IF EXISTS eql_v1.remove_index(table_name text, column_name text, index_name text);
159159

160160
CREATE FUNCTION eql_v1.remove_index(table_name text, column_name text, index_name text)
161161
RETURNS jsonb
@@ -216,7 +216,7 @@ AS $$
216216
$$ LANGUAGE plpgsql;
217217

218218

219-
DROP FUNCTION IF EXISTS eql_v1.modify_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
219+
-- DROP FUNCTION IF EXISTS eql_v1.modify_index(table_name text, column_name text, index_name text, cast_as text, opts jsonb);
220220

221221
CREATE FUNCTION eql_v1.modify_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
222222
RETURNS jsonb
@@ -240,7 +240,7 @@ $$ LANGUAGE plpgsql;
240240
--
241241
-- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
242242
--
243-
DROP FUNCTION IF EXISTS eql_v1.encrypt();
243+
-- DROP FUNCTION IF EXISTS eql_v1.encrypt();
244244

245245
CREATE FUNCTION eql_v1.encrypt(force boolean DEFAULT false)
246246
RETURNS boolean
@@ -267,7 +267,7 @@ AS $$
267267
$$ LANGUAGE plpgsql;
268268

269269

270-
DROP FUNCTION IF EXISTS eql_v1.activate();
270+
-- DROP FUNCTION IF EXISTS eql_v1.activate();
271271

272272
CREATE FUNCTION eql_v1.activate()
273273
RETURNS boolean
@@ -285,7 +285,7 @@ AS $$
285285
$$ LANGUAGE plpgsql;
286286

287287

288-
DROP FUNCTION IF EXISTS eql_v1.discard();
288+
-- DROP FUNCTION IF EXISTS eql_v1.discard();
289289

290290
CREATE FUNCTION eql_v1.discard()
291291
RETURNS boolean
@@ -301,7 +301,7 @@ AS $$
301301
$$ LANGUAGE plpgsql;
302302

303303

304-
DROP FUNCTION IF EXISTS eql_v1.add_column(table_name text, column_name text, cast_as text);
304+
-- DROP FUNCTION IF EXISTS eql_v1.add_column(table_name text, column_name text, cast_as text);
305305

306306
CREATE FUNCTION eql_v1.add_column(table_name text, column_name text, cast_as text DEFAULT 'text')
307307
RETURNS jsonb
@@ -340,7 +340,7 @@ AS $$
340340
$$ LANGUAGE plpgsql;
341341

342342

343-
DROP FUNCTION IF EXISTS eql_v1.remove_column(table_name text, column_name text);
343+
-- DROP FUNCTION IF EXISTS eql_v1.remove_column(table_name text, column_name text);
344344

345345
CREATE FUNCTION eql_v1.remove_column(table_name text, column_name text)
346346
RETURNS jsonb
@@ -396,7 +396,7 @@ AS $$
396396
$$ LANGUAGE plpgsql;
397397

398398

399-
DROP FUNCTION IF EXISTS eql_v1.reload_config();
399+
-- DROP FUNCTION IF EXISTS eql_v1.reload_config();
400400

401401
CREATE FUNCTION eql_v1.reload_config()
402402
RETURNS void
@@ -405,7 +405,7 @@ BEGIN ATOMIC
405405
RETURN NULL;
406406
END;
407407

408-
DROP FUNCTION IF EXISTS eql_v1.config();
408+
-- DROP FUNCTION IF EXISTS eql_v1.config();
409409

410410
-- A convenience function to return the configuration in a tabular format, allowing for easier filtering, and querying.
411411
-- Query using `SELECT * FROM cs_config();`

‎src/config/tables.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
--
44
--
5-
-- CREATE the cs_configuration_v1 TABLE
5+
-- CREATE the eql_v1_configuration TABLE
66
--
77
CREATE TABLE IF NOT EXISTS public.eql_v1_configuration
88
(

‎src/encrypted/aggregates.sql

+4-4
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
-- REQUIRE: src/ore/functions.sql
44

55
-- Aggregate functions for ORE
6-
DROP AGGREGATE IF EXISTS eql_v1.min(eql_v1_encrypted);
7-
DROP FUNCTION IF EXISTS eql_v1.min(a eql_v1_encrypted, b eql_v1_encrypted);
6+
-- DROP AGGREGATE IF EXISTS eql_v1.min(eql_v1_encrypted);
7+
-- DROP FUNCTION IF EXISTS eql_v1.min(a eql_v1_encrypted, b eql_v1_encrypted);
88

99
CREATE FUNCTION eql_v1.min(a eql_v1_encrypted, b eql_v1_encrypted)
1010
RETURNS eql_v1_encrypted
@@ -27,8 +27,8 @@ CREATE AGGREGATE eql_v1.min(eql_v1_encrypted)
2727
stype = eql_v1_encrypted
2828
);
2929

30-
DROP AGGREGATE IF EXISTS eql_v1.max(eql_v1_encrypted);
31-
DROP FUNCTION IF EXISTS eql_v1.max(a eql_v1_encrypted, b eql_v1_encrypted);
30+
-- DROP AGGREGATE IF EXISTS eql_v1.max(eql_v1_encrypted);
31+
-- DROP FUNCTION IF EXISTS eql_v1.max(a eql_v1_encrypted, b eql_v1_encrypted);
3232

3333
CREATE FUNCTION eql_v1.max(a eql_v1_encrypted, b eql_v1_encrypted)
3434
RETURNS eql_v1_encrypted

‎src/encrypted/aggregates_test.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
\set ON_ERROR_STOP on
22

33
-- create table
4-
DROP TABLE IF EXISTS agg_test;
4+
-- DROP TABLE IF EXISTS agg_test;
55
CREATE TABLE agg_test
66
(
77
plain_int integer,

0 commit comments

Comments
 (0)