From 8bd06591e8a40221593f4519fe9dc273640836ca Mon Sep 17 00:00:00 2001 From: Domino Valdano <2644901+reductionista@users.noreply.github.com> Date: Thu, 31 Oct 2024 17:02:44 -0700 Subject: [PATCH 1/8] Add Solana db schema --- .../migrations/0259_add_solana_schema.sql | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 core/store/migrate/migrations/0259_add_solana_schema.sql diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql new file mode 100644 index 00000000000..6fe193b150e --- /dev/null +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -0,0 +1,50 @@ +-- +goose Up +-- +goose StatementBegin +-- This removes evm from the search path, to avoid risk of an unqualified query intended to act on a Solana table accidentally +-- acting on the corresponding evm table. This makes schema qualification mandatory for all chain-specific queries. +SET search_path TO public; +CREATE SCHEMA solana + +CREATE TABLE solana.log_poller_filters ( + id BIGSERIAL + chain_id TEXT + name TEXT NOT NULL + address BYTEA NOT NULL + event_name TEXT NOT NULL + event_sig BYTEA NOT NULL + starting_block BIGINT NOT NULL + event_idl TEXT + sub_key_paths TEXT[][] -- A list of subkeys to be indexed, represented by their json paths in the event struct + retention BIGINT NOT NULL DEFAULT 0 -– we don’t have to implement this initially, but good to include it in the schema + max_logs_kept BIGINT NOT NULL DEFAULT 0 -- same as retention, no need to implement yet +) + +CREATE TABLE solana.logs ( + id BIGSERIAL + filter_id BIGINT NOT NULL REFERENCES solana.log_poller_filters (id) ON DELETE CASCADE + chain_id TEXT not null + log_index bigint not null + block_hash bytea not null + block_number bigint not null + block_timestamp timestamp with time zone not null + address bytea not null + event_sig bytea not null + subkey_values bytea[] not null + tx_hash bytea not null + data bytea not null + created_at timestamp with time zone not null + expires_at timestamp with time zone not null + sequence_num bigint not null +) + +-- +goose StatementEnd + + +-- +goose Down +-- +goose StatementBegin + +DROP TABLE IF EXISTS solana.logs +DROP TABLE IF EXISTS solana.log_poller_filters + +DROP SCHEMA solana; +-- +goose StatementEnd From 46edfaed91f4aca4ff9037b12a9dd70126c9bf69 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 8 Nov 2024 19:55:04 +0100 Subject: [PATCH 2/8] solana db schema --- .../migrations/0259_add_solana_schema.sql | 77 ++++++++++++------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql index 6fe193b150e..297b4a101d4 100644 --- a/core/store/migrate/migrations/0259_add_solana_schema.sql +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -3,39 +3,56 @@ -- This removes evm from the search path, to avoid risk of an unqualified query intended to act on a Solana table accidentally -- acting on the corresponding evm table. This makes schema qualification mandatory for all chain-specific queries. SET search_path TO public; -CREATE SCHEMA solana +CREATE SCHEMA solana; CREATE TABLE solana.log_poller_filters ( - id BIGSERIAL - chain_id TEXT - name TEXT NOT NULL - address BYTEA NOT NULL - event_name TEXT NOT NULL - event_sig BYTEA NOT NULL - starting_block BIGINT NOT NULL - event_idl TEXT - sub_key_paths TEXT[][] -- A list of subkeys to be indexed, represented by their json paths in the event struct - retention BIGINT NOT NULL DEFAULT 0 -– we don’t have to implement this initially, but good to include it in the schema + id BIGSERIAL PRIMARY KEY, + chain_id TEXT NOT NULL, -- use human-readable name instead of genesis block hash to reduce index size by 50% + name TEXT NOT NULL, + address BYTEA NOT NULL, + event_name TEXT NOT NULL, + event_sig BYTEA NOT NULL, + starting_block BIGINT NOT NULL, + event_idl TEXT, + sub_key_paths TEXT[][], -- A list of subkeys to be indexed, represented by their json paths in the event struct + retention BIGINT NOT NULL DEFAULT 0, -- we don’t have to implement this initially, but good to include it in the schema max_logs_kept BIGINT NOT NULL DEFAULT 0 -- same as retention, no need to implement yet -) +); CREATE TABLE solana.logs ( - id BIGSERIAL - filter_id BIGINT NOT NULL REFERENCES solana.log_poller_filters (id) ON DELETE CASCADE - chain_id TEXT not null - log_index bigint not null - block_hash bytea not null - block_number bigint not null - block_timestamp timestamp with time zone not null - address bytea not null - event_sig bytea not null - subkey_values bytea[] not null - tx_hash bytea not null - data bytea not null - created_at timestamp with time zone not null - expires_at timestamp with time zone not null + id BIGSERIAL PRIMARY KEY, + filter_id BIGINT NOT NULL REFERENCES solana.log_poller_filters (id) ON DELETE CASCADE, + chain_id TEXT not null, -- use human-readable name instead of genesis block hash to reduce index size by 50% + log_index bigint not null, + block_hash bytea not null, + block_number bigint not null CHECK (block_number > 0), + block_timestamp timestamp with time zone not null, + address bytea not null, + event_sig bytea not null, + subkey_values bytea[] not null, + tx_hash bytea not null, + data bytea not null, + created_at timestamp with time zone not null, + expires_at timestamp with time zone null, -- null to indicate no timebase expiry sequence_num bigint not null -) +); + +CREATE INDEX IF NOT EXISTS solana_logs_idx_by_timestamp ON solana.logs (chain_id, address, event_sig, block_timestamp, block_number); + +CREATE INDEX IF NOT EXISTS solana_logs_idx ON solana.logs (chain_id, block_number, address, event_sig); + +CREATE INDEX IF NOT EXISTS solana_logs_idx_subkey_one ON solana.logs ((subkey_values[1])); +CREATE INDEX IF NOT EXISTS solana_logs_idx_subkey_two ON solana.logs ((subkey_values[2])); +CREATE INDEX IF NOT EXISTS solana_logs_idx_subkey_three ON solana.logs ((subkey_values[3])); +CREATE INDEX IF NOT EXISTS solana_logs_idx_subkey_four ON solana.logs ((subkey_values[4])); + +CREATE INDEX IF NOT EXISTS solana_logs_idx_tx_hash ON solana.logs (tx_hash); + +-- Useful for the current form of those queries: WHERE chain_id = $1 AND address = $2 AND event_sig = $3 ... ORDER BY block_number, log_index +-- log_index is not included in this index because it increases the index size by ~ 10% for a likely negligible performance benefit +CREATE INDEX IF NOT EXISTS solana_logs_idx_chain_address_event_block ON solana.logs (chain_id, address, event_sig, block_number); + +CREATE UNIQUE INDEX IF NOT EXISTS solana_logs_idx_chain_filter_block_logindex ON solana.logs USING btree (chain_id, filter_id, block_number, log_index); -- +goose StatementEnd @@ -43,8 +60,10 @@ CREATE TABLE solana.logs ( -- +goose Down -- +goose StatementBegin -DROP TABLE IF EXISTS solana.logs -DROP TABLE IF EXISTS solana.log_poller_filters +DROP TABLE IF EXISTS solana.logs; +DROP TABLE IF EXISTS solana.log_poller_filters; DROP SCHEMA solana; + +SET search_path TO public,evm; -- +goose StatementEnd From 5721a58ae4e93882f5053c2b567e2dea9eeef3f8 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 13 Nov 2024 20:32:28 +0100 Subject: [PATCH 3/8] add missing index on solana filters --- .../migrate/migrations/0259_add_solana_schema.sql | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql index 297b4a101d4..c9e86ad43b4 100644 --- a/core/store/migrate/migrations/0259_add_solana_schema.sql +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -14,11 +14,22 @@ CREATE TABLE solana.log_poller_filters ( event_sig BYTEA NOT NULL, starting_block BIGINT NOT NULL, event_idl TEXT, - sub_key_paths TEXT[][], -- A list of subkeys to be indexed, represented by their json paths in the event struct + sub_key_paths json, -- A list of subkeys to be indexed, represented by their json paths in the event struct. Forced to use json's 2d array as text[][] requires all paths to have equal length. retention BIGINT NOT NULL DEFAULT 0, -- we don’t have to implement this initially, but good to include it in the schema max_logs_kept BIGINT NOT NULL DEFAULT 0 -- same as retention, no need to implement yet ); +-- This generates a unique BIGINT for the log_poller_filters table from hashing (name, chain_id, address, event_sig, sub_key_paths). +-- Using an ordinary multi-column index on 5 columns would require a lot of extra storage space. +-- Note for updating this if and when we drop support for postgresql 12 & 13: hashrecordextended() can be used directly in postgresql 14, avoiding the need for a helper function. +-- The helper function is necessary only for the IMMUTABLE keyword. +CREATE OR REPLACE FUNCTION solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, sub_key_paths json) + RETURNS BIGINT + LANGUAGE SQL IMMUTABLE COST 25 PARALLEL SAFE AS 'SELECT hashtextextended(textin(record_out(($1,$2,$3,$4,$5))), 0)'; + +CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filters_hash_key + ON solana.log_poller_filters (solana.f_log_poller_filter_hash(name, chain_id, address, event_sig, sub_key_paths)); + CREATE TABLE solana.logs ( id BIGSERIAL PRIMARY KEY, filter_id BIGINT NOT NULL REFERENCES solana.log_poller_filters (id) ON DELETE CASCADE, @@ -63,6 +74,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS solana_logs_idx_chain_filter_block_logindex ON DROP TABLE IF EXISTS solana.logs; DROP TABLE IF EXISTS solana.log_poller_filters; +DROP FUNCTION IF EXISTS solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, sub_key_paths json); + DROP SCHEMA solana; SET search_path TO public,evm; From 100f4d7083291f47631cf350abb9129ab584997d Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 14 Nov 2024 14:27:09 +0100 Subject: [PATCH 4/8] ensure subkey naming is consistent --- .../migrate/migrations/0259_add_solana_schema.sql | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql index c9e86ad43b4..9d538e5bbf7 100644 --- a/core/store/migrate/migrations/0259_add_solana_schema.sql +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -14,21 +14,23 @@ CREATE TABLE solana.log_poller_filters ( event_sig BYTEA NOT NULL, starting_block BIGINT NOT NULL, event_idl TEXT, - sub_key_paths json, -- A list of subkeys to be indexed, represented by their json paths in the event struct. Forced to use json's 2d array as text[][] requires all paths to have equal length. + subkey_paths json, -- A list of subkeys to be indexed, represented by their json paths in the event struct. Forced to use json's 2d array as text[][] requires all paths to have equal length. retention BIGINT NOT NULL DEFAULT 0, -- we don’t have to implement this initially, but good to include it in the schema max_logs_kept BIGINT NOT NULL DEFAULT 0 -- same as retention, no need to implement yet ); --- This generates a unique BIGINT for the log_poller_filters table from hashing (name, chain_id, address, event_sig, sub_key_paths). +-- This generates a unique BIGINT for the log_poller_filters table from hashing (name, chain_id, address, event_sig, subkey_paths). -- Using an ordinary multi-column index on 5 columns would require a lot of extra storage space. -- Note for updating this if and when we drop support for postgresql 12 & 13: hashrecordextended() can be used directly in postgresql 14, avoiding the need for a helper function. -- The helper function is necessary only for the IMMUTABLE keyword. -CREATE OR REPLACE FUNCTION solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, sub_key_paths json) +CREATE OR REPLACE FUNCTION solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, subkey_paths json) RETURNS BIGINT LANGUAGE SQL IMMUTABLE COST 25 PARALLEL SAFE AS 'SELECT hashtextextended(textin(record_out(($1,$2,$3,$4,$5))), 0)'; CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filters_hash_key - ON solana.log_poller_filters (solana.f_log_poller_filter_hash(name, chain_id, address, event_sig, sub_key_paths)); + ON solana.log_poller_filters (solana.f_log_poller_filter_hash(name, chain_id, address, event_sig, subkey_paths)); + +CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filter_name ON solana.log_poller_filters (name); CREATE TABLE solana.logs ( id BIGSERIAL PRIMARY KEY, @@ -74,7 +76,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS solana_logs_idx_chain_filter_block_logindex ON DROP TABLE IF EXISTS solana.logs; DROP TABLE IF EXISTS solana.log_poller_filters; -DROP FUNCTION IF EXISTS solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, sub_key_paths json); +DROP FUNCTION IF EXISTS solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, subkey_paths json); DROP SCHEMA solana; From d4281bf2e6570ea35f77bead9dd11422a5391f75 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 14 Nov 2024 18:49:22 +0100 Subject: [PATCH 5/8] remove redundant constraint --- .../migrate/migrations/0259_add_solana_schema.sql | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql index 9d538e5bbf7..ef863348fbb 100644 --- a/core/store/migrate/migrations/0259_add_solana_schema.sql +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -19,17 +19,6 @@ CREATE TABLE solana.log_poller_filters ( max_logs_kept BIGINT NOT NULL DEFAULT 0 -- same as retention, no need to implement yet ); --- This generates a unique BIGINT for the log_poller_filters table from hashing (name, chain_id, address, event_sig, subkey_paths). --- Using an ordinary multi-column index on 5 columns would require a lot of extra storage space. --- Note for updating this if and when we drop support for postgresql 12 & 13: hashrecordextended() can be used directly in postgresql 14, avoiding the need for a helper function. --- The helper function is necessary only for the IMMUTABLE keyword. -CREATE OR REPLACE FUNCTION solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, subkey_paths json) - RETURNS BIGINT - LANGUAGE SQL IMMUTABLE COST 25 PARALLEL SAFE AS 'SELECT hashtextextended(textin(record_out(($1,$2,$3,$4,$5))), 0)'; - -CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filters_hash_key - ON solana.log_poller_filters (solana.f_log_poller_filter_hash(name, chain_id, address, event_sig, subkey_paths)); - CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filter_name ON solana.log_poller_filters (name); CREATE TABLE solana.logs ( @@ -76,8 +65,6 @@ CREATE UNIQUE INDEX IF NOT EXISTS solana_logs_idx_chain_filter_block_logindex ON DROP TABLE IF EXISTS solana.logs; DROP TABLE IF EXISTS solana.log_poller_filters; -DROP FUNCTION IF EXISTS solana.f_log_poller_filter_hash(name TEXT, chain_id TEXT, address BYTEA, event_sig BYTEA, subkey_paths json); - DROP SCHEMA solana; SET search_path TO public,evm; From cd966dfdb0542d4e39c79e1f7b8317e3ec04ca6f Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 21 Nov 2024 16:35:16 +0100 Subject: [PATCH 6/8] added is_deleted column to solana.log_poller_filters --- core/store/migrate/migrations/0259_add_solana_schema.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql index ef863348fbb..8906d31f3f3 100644 --- a/core/store/migrate/migrations/0259_add_solana_schema.sql +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -16,10 +16,11 @@ CREATE TABLE solana.log_poller_filters ( event_idl TEXT, subkey_paths json, -- A list of subkeys to be indexed, represented by their json paths in the event struct. Forced to use json's 2d array as text[][] requires all paths to have equal length. retention BIGINT NOT NULL DEFAULT 0, -- we don’t have to implement this initially, but good to include it in the schema - max_logs_kept BIGINT NOT NULL DEFAULT 0 -- same as retention, no need to implement yet + max_logs_kept BIGINT NOT NULL DEFAULT 0, -- same as retention, no need to implement yet + is_deleted BOOLEAN NOT NULL DEFAULT FALSE ); -CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filter_name ON solana.log_poller_filters (name); +CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filter_name ON solana.log_poller_filters (name) WHERE NOT is_deleted; CREATE TABLE solana.logs ( id BIGSERIAL PRIMARY KEY, From b20c1816ce3dcfb745fc020e26ce5bfc1eb03c12 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 26 Nov 2024 15:43:39 +0100 Subject: [PATCH 7/8] fix solana logpoller filter name constraint --- core/store/migrate/migrations/0259_add_solana_schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0259_add_solana_schema.sql index 8906d31f3f3..035d3ff3d61 100644 --- a/core/store/migrate/migrations/0259_add_solana_schema.sql +++ b/core/store/migrate/migrations/0259_add_solana_schema.sql @@ -20,7 +20,7 @@ CREATE TABLE solana.log_poller_filters ( is_deleted BOOLEAN NOT NULL DEFAULT FALSE ); -CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filter_name ON solana.log_poller_filters (name) WHERE NOT is_deleted; +CREATE UNIQUE INDEX IF NOT EXISTS solana_log_poller_filter_name ON solana.log_poller_filters (chain_id, name) WHERE NOT is_deleted; CREATE TABLE solana.logs ( id BIGSERIAL PRIMARY KEY, From db4da79894c445a10cebac5df72e3f394d2a4f32 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 18 Dec 2024 13:23:39 +0100 Subject: [PATCH 8/8] bump migration number --- .../{0259_add_solana_schema.sql => 0262_add_solana_schema.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename core/store/migrate/migrations/{0259_add_solana_schema.sql => 0262_add_solana_schema.sql} (100%) diff --git a/core/store/migrate/migrations/0259_add_solana_schema.sql b/core/store/migrate/migrations/0262_add_solana_schema.sql similarity index 100% rename from core/store/migrate/migrations/0259_add_solana_schema.sql rename to core/store/migrate/migrations/0262_add_solana_schema.sql