From 2e1536a594020dc9fa151d20652e5360a978f331 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 22 Oct 2024 07:54:23 +0200 Subject: [PATCH] feat: add missing migrations to clean database --- .../migrations/19-clean-database/notes.yaml | 1 + .../migrations/19-clean-database/up.sql | 76 +++++++++++++++++++ internal/storage/ledger/balances.go | 4 +- internal/storage/ledger/transactions.go | 2 +- 4 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 internal/storage/bucket/migrations/19-clean-database/notes.yaml create mode 100644 internal/storage/bucket/migrations/19-clean-database/up.sql diff --git a/internal/storage/bucket/migrations/19-clean-database/notes.yaml b/internal/storage/bucket/migrations/19-clean-database/notes.yaml new file mode 100644 index 000000000..759af9421 --- /dev/null +++ b/internal/storage/bucket/migrations/19-clean-database/notes.yaml @@ -0,0 +1 @@ +name: Clean not used columns in database diff --git a/internal/storage/bucket/migrations/19-clean-database/up.sql b/internal/storage/bucket/migrations/19-clean-database/up.sql new file mode 100644 index 000000000..39597ae54 --- /dev/null +++ b/internal/storage/bucket/migrations/19-clean-database/up.sql @@ -0,0 +1,76 @@ +set search_path = '{{.Bucket}}'; + +-- Clean all useless function/aggregates/indexes inherited from stateful version. +drop aggregate aggregate_objects(jsonb); +drop aggregate first(anyelement); + +drop function array_distinct(anyarray); +drop function insert_posting(_transaction_seq bigint, _ledger character varying, _insertion_date timestamp without time zone, _effective_date timestamp without time zone, posting jsonb, _account_metadata jsonb); +drop function upsert_account(_ledger character varying, _address character varying, _metadata jsonb, _date timestamp without time zone, _first_usage timestamp without time zone); +drop function get_latest_move_for_account_and_asset(_ledger character varying, _account_address character varying, _asset character varying, _before timestamp without time zone); +drop function update_transaction_metadata(_ledger character varying, _id numeric, _metadata jsonb, _date timestamp without time zone); +drop function delete_account_metadata(_ledger character varying, _address character varying, _key character varying, _date timestamp without time zone); +drop function delete_transaction_metadata(_ledger character varying, _id numeric, _key character varying, _date timestamp without time zone); +drop function balance_from_volumes(v volumes); +drop function get_all_account_volumes(_ledger character varying, _account character varying, _before timestamp without time zone); +drop function first_agg(anyelement, anyelement); +drop function volumes_to_jsonb(v volumes_with_asset); +drop function get_account_aggregated_effective_volumes(_ledger character varying, _account_address character varying, _before timestamp without time zone); +drop function handle_log(); +drop function get_account_aggregated_volumes(_ledger character varying, _account_address character varying, _before timestamp without time zone); +drop function get_aggregated_volumes_for_transaction(_ledger character varying, tx numeric); +drop function insert_move(_transactions_seq bigint, _ledger character varying, _insertion_date timestamp without time zone, _effective_date timestamp without time zone, _account_address character varying, _asset character varying, _amount numeric, _is_source boolean, _account_exists boolean); +drop function get_all_assets(_ledger character varying); +drop function insert_transaction(_ledger character varying, data jsonb, _date timestamp without time zone, _account_metadata jsonb); +drop function get_all_account_effective_volumes(_ledger character varying, _account character varying, _before timestamp without time zone); +drop function get_account_balance(_ledger character varying, _account character varying, _asset character varying, _before timestamp without time zone); +drop function get_aggregated_effective_volumes_for_transaction(_ledger character varying, tx numeric); +drop function aggregate_ledger_volumes(_ledger character varying, _before timestamp without time zone, _accounts character varying[], _assets character varying[] ); +drop function get_transaction(_ledger character varying, _id numeric, _before timestamp without time zone); +drop function revert_transaction(_ledger character varying, _id numeric, _date timestamp without time zone); + +drop index transactions_sources_arrays; +drop index transactions_destinations_arrays; +drop index accounts_address_array; +drop index accounts_address_array_length; +drop index transactions_sources; +drop index transactions_destinations; + +-- We will remove some triggers writing these columns (set_compat_xxx) later in this file. +-- When these triggers will be removed, there is a little moment where the columns will not be filled and constraints +-- still checked by the database. +-- So, we drop the not null constraint before removing the triggers. +-- Once the triggers removed, we will be able to drop the columns. +alter table moves +alter column transactions_seq drop not null, +alter column accounts_seq drop not null, +alter column accounts_address_array drop not null; + +alter table transactions_metadata +alter column transactions_seq drop not null; + +alter table accounts_metadata +alter column accounts_seq drop not null; + +-- Now, the columns are nullable, we can drop the trigger +drop trigger set_compat_on_move on moves; +drop trigger set_compat_on_accounts_metadata on accounts_metadata; +drop trigger set_compat_on_transactions_metadata on transactions_metadata; +drop function set_compat_on_move(); +drop function set_compat_on_accounts_metadata(); +drop function set_compat_on_transactions_metadata(); + +-- Finally remove the columns +alter table moves +drop column transactions_seq, +drop column accounts_seq, +drop column accounts_address_array; + +alter table transactions_metadata +drop column transactions_seq; + +alter table accounts_metadata +drop column accounts_seq; + +alter table transactions +drop column seq; \ No newline at end of file diff --git a/internal/storage/ledger/balances.go b/internal/storage/ledger/balances.go index 566e45d76..b79c38080 100644 --- a/internal/storage/ledger/balances.go +++ b/internal/storage/ledger/balances.go @@ -154,7 +154,7 @@ func (s *Store) selectAccountWithAggregatedVolumes(date *time.Time, useInsertion TableExpr("(?) values", selectAccountWithAssetAndVolumes). Group("accounts_address"). Column("accounts_address"). - ColumnExpr("aggregate_objects(json_build_object(asset, json_build_object('input', (volumes).inputs, 'output', (volumes).outputs))::jsonb) as " + alias) + ColumnExpr("public.aggregate_objects(json_build_object(asset, json_build_object('input', (volumes).inputs, 'output', (volumes).outputs))::jsonb) as " + alias) } func (s *Store) SelectAggregatedBalances(date *time.Time, useInsertionDate bool, builder query.Builder) *bun.SelectQuery { @@ -168,7 +168,7 @@ func (s *Store) SelectAggregatedBalances(date *time.Time, useInsertionDate bool, return s.db.NewSelect(). TableExpr("(?) values", sumVolumesForAsset). - ColumnExpr("aggregate_objects(json_build_object(asset, volumes)::jsonb) as aggregated") + ColumnExpr("public.aggregate_objects(json_build_object(asset, volumes)::jsonb) as aggregated") } func (s *Store) GetAggregatedBalances(ctx context.Context, q ledgercontroller.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { diff --git a/internal/storage/ledger/transactions.go b/internal/storage/ledger/transactions.go index 5f001ff11..ef92e887d 100644 --- a/internal/storage/ledger/transactions.go +++ b/internal/storage/ledger/transactions.go @@ -158,7 +158,7 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti `), ). Column("transactions_id"). - ColumnExpr("aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes"). + ColumnExpr("public.aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes"). Group("transactions_id"), ). ColumnExpr("pcev.*")