From 8742889f3a72699b23780995232431a40e4382b0 Mon Sep 17 00:00:00 2001 From: iljagrabar14 Date: Mon, 22 Jul 2024 09:11:00 +0200 Subject: [PATCH 1/2] Init migrations for new tables --- .../000002_edit_validator_message.sql | 25 ++++++++++++++ ...03_cancel_unbonding_delegation_message.sql | 26 ++++++++++++++ .../clickhouse/000004_delegate_message.sql | 31 +++++++++++++++++ .../000005_create_validator_message.sql | 34 +++++++++++++++++++ .../000006_unbonding_delegation_message.sql | 30 ++++++++++++++++ .../000007_begin_redelegate_message.sql | 32 +++++++++++++++++ migrations/clickhouse/000008_message.sql | 28 +++++++++++++++ 7 files changed, 206 insertions(+) create mode 100644 migrations/clickhouse/000002_edit_validator_message.sql create mode 100644 migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql create mode 100644 migrations/clickhouse/000004_delegate_message.sql create mode 100644 migrations/clickhouse/000005_create_validator_message.sql create mode 100644 migrations/clickhouse/000006_unbonding_delegation_message.sql create mode 100644 migrations/clickhouse/000007_begin_redelegate_message.sql create mode 100644 migrations/clickhouse/000008_message.sql diff --git a/migrations/clickhouse/000002_edit_validator_message.sql b/migrations/clickhouse/000002_edit_validator_message.sql new file mode 100644 index 0000000..1eba734 --- /dev/null +++ b/migrations/clickhouse/000002_edit_validator_message.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS spacebox.edit_validator_message +( + `timestamp` DateTime, + `height` Int64, + `tx_hash` String, + `description` String +) ENGINE = MergeTree() + ORDER BY (`timestamp`, `height`, `tx_hash`); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.edit_validator_message_writer +TO spacebox.edit_validator_message AS +select + height, + `timestamp`, + txhash as tx_hash, + JSONExtractString(msg, 'description') as description +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction + where type = '/cosmos.staking.v1beta1.MsgEditValidator' +) diff --git a/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql b/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql new file mode 100644 index 0000000..118de23 --- /dev/null +++ b/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS spacebox.cancel_unbonding_delegation_message +( + `timestamp` DateTime, + `height` Int64, + `tx_hash` String, + `delegator_address` String, + `validator_address` String +) ENGINE = MergeTree() + ORDER BY (`timestamp`, `height`, `tx_hash`, `delegator_address`, `validator_address`); + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.cancel_unbonding_delegation_message_writer +TO spacebox.cancel_unbonding_delegation_message AS +select + height, + `timestamp`, + txhash as tx_hash, + JSONExtractString(msg, 'delegatorAddress') AS delegator_address, + JSONExtractString(msg, 'validatorAddress') AS validator_address +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction + where type = '/cosmos.staking.v1beta1.MsgCancelUnbondingDelegation' +) diff --git a/migrations/clickhouse/000004_delegate_message.sql b/migrations/clickhouse/000004_delegate_message.sql new file mode 100644 index 0000000..195a6d6 --- /dev/null +++ b/migrations/clickhouse/000004_delegate_message.sql @@ -0,0 +1,31 @@ +CREATE TABLE IF NOT EXISTS spacebox.delegation_message +( + `timestamp` DateTime, + `operator_address` String, + `delegator_address` String, + `coin` String, + `amount` Int256, + `height` Int64, + `tx_hash` String +) ENGINE = MergeTree() + ORDER BY (`operator_address`, `delegator_address`, `height`, `tx_hash`, `timestamp`); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.delegation_message_writer +TO spacebox.delegation_message AS +select + height, + `timestamp`, + txhash as tx_hash, + JSONExtractString(msg, 'delegatorAddress') as delegator_address, + JSONExtractString(msg, 'validatorAddress') as operator_address, + JSONExtractString(JSONExtractString(msg, 'amount'), 'denom') as coin, + toInt256(JSONExtractString(JSONExtractString(msg, 'amount'), 'amount')) as amount +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction + where type = '/cosmos.staking.v1beta1.MsgDelegate' +) diff --git a/migrations/clickhouse/000005_create_validator_message.sql b/migrations/clickhouse/000005_create_validator_message.sql new file mode 100644 index 0000000..718e33d --- /dev/null +++ b/migrations/clickhouse/000005_create_validator_message.sql @@ -0,0 +1,34 @@ +CREATE TABLE IF NOT EXISTS spacebox.create_validator_message +( + `timestamp` DateTime + `height` Int64, + `tx_hash` String, + `delegator_address` String, + `validator_address` String, + `description` String, + `commission` String, + `min_self_delegation` Int64, + `pubkey` String +) ENGINE = MergeTree() + ORDER BY (`timestamp`, `height`, `tx_hash`, `delegator_address`, `validator_address`, `commission_rates` ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.create_validator_message_writer +TO spacebox.create_validator_message AS +select + height, + `timestamp`, + txhash as tx_hash, + JSONExtractString(msg, 'delegatorAddress') as delegator_address, + JSONExtractString(msg, 'validatorAddress') as validator_address, + JSONExtractString(msg, 'description') as description, + JSONExtractString(msg, 'commission') as commission, + JSONExtractString(msg, 'pubkey') as pubkey, + toInt64(JSONExtractString(msg, 'minSelfDelegation')) as min_self_delegation +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction + where type = '/cosmos.staking.v1beta1.MsgCreateValidator' +) diff --git a/migrations/clickhouse/000006_unbonding_delegation_message.sql b/migrations/clickhouse/000006_unbonding_delegation_message.sql new file mode 100644 index 0000000..cebc185 --- /dev/null +++ b/migrations/clickhouse/000006_unbonding_delegation_message.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS spacebox.unbonding_delegation_message +( + `timestamp` DateTime + `operator_address` String, + `delegator_address` String, + `amount` Int256, + `completion_time` DateTime, + `height` Int64, + `tx_hash` String +) ENGINE = MergeTree() + ORDER BY (`timestamp`, `tx_hash`, `operator_address`, `delegator_address`, `completion_time`, `height`); + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.unbonding_delegation_message_writer +TO spacebox.unbonding_delegation_message AS +select + height, + `timestamp`, + txhash as tx_hash, + JSONExtractString(msg, 'delegatorAddress') as delegator_address, + JSONExtractString(msg, 'validatorAddress') as operator_address, + toInt256(JSONExtractString(JSONExtractString(msg, 'amount'), 'amount')) as amount, + parseDateTimeBestEffort(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'key') = 'completion_time', JSONExtractArrayRaw(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'type') = 'unbond', JSONExtractArrayRaw(events))[1], 'attributes')))[1], 'value')) AS completion_time +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction + where type = '/cosmos.staking.v1beta1.MsgUndelegate' +) diff --git a/migrations/clickhouse/000007_begin_redelegate_message.sql b/migrations/clickhouse/000007_begin_redelegate_message.sql new file mode 100644 index 0000000..a92664e --- /dev/null +++ b/migrations/clickhouse/000007_begin_redelegate_message.sql @@ -0,0 +1,32 @@ +CREATE TABLE IF NOT EXISTS spacebox.redelegation_message +( + `delegator_address` String, + `src_validator_address` String, + `dst_validator_address` String, + `amount` Int256, + `height` Int64, + `completion_time` DateTime, + `tx_hash` String, + `timestamp` DateTime +) ENGINE = MergeTree() + ORDER BY (`delegator_address`, `src_validator_address`, `dst_validator_address`, `height`, `completion_time`, `tx_hash`, `timestamp`); + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.redelegation_message_writer +TO spacebox.redelegation_message AS +select + height, + `timestamp`, + txhash as tx_hash, + JSONExtractString(msg, 'delegatorAddress') as delegator_address, + JSONExtractString(msg, 'validatorSrcAddress') as src_validator_address, + JSONExtractString(msg, 'validatorDstAddress') as dst_validator_address, + toInt256(JSONExtractString(JSONExtractString(msg, 'amount'), 'amount')) as amount, + parseDateTimeBestEffort(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'key') = 'completion_time', JSONExtractArrayRaw(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'type') = 'redelegate', JSONExtractArrayRaw(events))[1], 'attributes')))[1], 'value')) AS completion_time +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction + where type = '/cosmos.staking.v1beta1.MsgBeginRedelegate' +) diff --git a/migrations/clickhouse/000008_message.sql b/migrations/clickhouse/000008_message.sql new file mode 100644 index 0000000..6d48beb --- /dev/null +++ b/migrations/clickhouse/000008_message.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS spacebox.message +( + `timestamp` DateTime, + `transaction_hash` String, + `msg_index` Int64, + `type` String, + `signer` String, + `value` String +) ENGINE = MergeTree() + ORDER BY (`transaction_hash`, `type`, `timestamp`, `signer`); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS spacebox.message_writer +TO spacebox.message AS +select + height, + `timestamp`, + type, + signer, + msg as value, + txhash as transaction_hash +from ( + select + *, + arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, + JSONExtractString(msg, '@type') AS type + from spacebox.raw_transaction +) \ No newline at end of file From d408d5191ec074a8ebf32604fbe84934726e0271 Mon Sep 17 00:00:00 2001 From: iljagrabar14 Date: Mon, 22 Jul 2024 16:26:54 +0200 Subject: [PATCH 2/2] Minor fixes --- migrations/clickhouse/000002_edit_validator_message.sql | 2 +- .../000003_cancel_unbonding_delegation_message.sql | 2 +- migrations/clickhouse/000004_delegate_message.sql | 8 +++----- migrations/clickhouse/000005_create_validator_message.sql | 2 +- .../clickhouse/000006_unbonding_delegation_message.sql | 6 +++--- migrations/clickhouse/000007_begin_redelegate_message.sql | 6 +++--- migrations/clickhouse/000008_message.sql | 1 + 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/migrations/clickhouse/000002_edit_validator_message.sql b/migrations/clickhouse/000002_edit_validator_message.sql index 1eba734..5ccd24f 100644 --- a/migrations/clickhouse/000002_edit_validator_message.sql +++ b/migrations/clickhouse/000002_edit_validator_message.sql @@ -21,5 +21,5 @@ from ( arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction - where type = '/cosmos.staking.v1beta1.MsgEditValidator' + where type = '/cosmos.staking.v1beta1.MsgEditValidator' and code = 0 ) diff --git a/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql b/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql index 118de23..d12a4b4 100644 --- a/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql +++ b/migrations/clickhouse/000003_cancel_unbonding_delegation_message.sql @@ -22,5 +22,5 @@ from ( arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction - where type = '/cosmos.staking.v1beta1.MsgCancelUnbondingDelegation' + where type = '/cosmos.staking.v1beta1.MsgCancelUnbondingDelegation' and code = 0 ) diff --git a/migrations/clickhouse/000004_delegate_message.sql b/migrations/clickhouse/000004_delegate_message.sql index 195a6d6..e4975c4 100644 --- a/migrations/clickhouse/000004_delegate_message.sql +++ b/migrations/clickhouse/000004_delegate_message.sql @@ -3,8 +3,7 @@ CREATE TABLE IF NOT EXISTS spacebox.delegation_message `timestamp` DateTime, `operator_address` String, `delegator_address` String, - `coin` String, - `amount` Int256, + `amount` String, `height` Int64, `tx_hash` String ) ENGINE = MergeTree() @@ -19,13 +18,12 @@ select txhash as tx_hash, JSONExtractString(msg, 'delegatorAddress') as delegator_address, JSONExtractString(msg, 'validatorAddress') as operator_address, - JSONExtractString(JSONExtractString(msg, 'amount'), 'denom') as coin, - toInt256(JSONExtractString(JSONExtractString(msg, 'amount'), 'amount')) as amount + JSONExtractString(msg, 'amount') as amount from ( select *, arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction - where type = '/cosmos.staking.v1beta1.MsgDelegate' + where type = '/cosmos.staking.v1beta1.MsgDelegate' and code = 0 ) diff --git a/migrations/clickhouse/000005_create_validator_message.sql b/migrations/clickhouse/000005_create_validator_message.sql index 718e33d..537ba9a 100644 --- a/migrations/clickhouse/000005_create_validator_message.sql +++ b/migrations/clickhouse/000005_create_validator_message.sql @@ -30,5 +30,5 @@ from ( arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction - where type = '/cosmos.staking.v1beta1.MsgCreateValidator' + where type = '/cosmos.staking.v1beta1.MsgCreateValidator' and code = 0 ) diff --git a/migrations/clickhouse/000006_unbonding_delegation_message.sql b/migrations/clickhouse/000006_unbonding_delegation_message.sql index cebc185..0403e1c 100644 --- a/migrations/clickhouse/000006_unbonding_delegation_message.sql +++ b/migrations/clickhouse/000006_unbonding_delegation_message.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS spacebox.unbonding_delegation_message `timestamp` DateTime `operator_address` String, `delegator_address` String, - `amount` Int256, + `amount` String, `completion_time` DateTime, `height` Int64, `tx_hash` String @@ -18,7 +18,7 @@ select txhash as tx_hash, JSONExtractString(msg, 'delegatorAddress') as delegator_address, JSONExtractString(msg, 'validatorAddress') as operator_address, - toInt256(JSONExtractString(JSONExtractString(msg, 'amount'), 'amount')) as amount, + JSONExtractString(msg, 'amount') as amount, parseDateTimeBestEffort(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'key') = 'completion_time', JSONExtractArrayRaw(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'type') = 'unbond', JSONExtractArrayRaw(events))[1], 'attributes')))[1], 'value')) AS completion_time from ( select @@ -26,5 +26,5 @@ from ( arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction - where type = '/cosmos.staking.v1beta1.MsgUndelegate' + where type = '/cosmos.staking.v1beta1.MsgUndelegate' and code = 0 ) diff --git a/migrations/clickhouse/000007_begin_redelegate_message.sql b/migrations/clickhouse/000007_begin_redelegate_message.sql index a92664e..b6ad766 100644 --- a/migrations/clickhouse/000007_begin_redelegate_message.sql +++ b/migrations/clickhouse/000007_begin_redelegate_message.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS spacebox.redelegation_message `delegator_address` String, `src_validator_address` String, `dst_validator_address` String, - `amount` Int256, + `amount` String, `height` Int64, `completion_time` DateTime, `tx_hash` String, @@ -20,7 +20,7 @@ select JSONExtractString(msg, 'delegatorAddress') as delegator_address, JSONExtractString(msg, 'validatorSrcAddress') as src_validator_address, JSONExtractString(msg, 'validatorDstAddress') as dst_validator_address, - toInt256(JSONExtractString(JSONExtractString(msg, 'amount'), 'amount')) as amount, + JSONExtractString(msg, 'amount') as amount, parseDateTimeBestEffort(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'key') = 'completion_time', JSONExtractArrayRaw(JSONExtractString(arrayFilter(x -> JSONExtractString(x, 'type') = 'redelegate', JSONExtractArrayRaw(events))[1], 'attributes')))[1], 'value')) AS completion_time from ( select @@ -28,5 +28,5 @@ from ( arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction - where type = '/cosmos.staking.v1beta1.MsgBeginRedelegate' + where type = '/cosmos.staking.v1beta1.MsgBeginRedelegate' and code = 0 ) diff --git a/migrations/clickhouse/000008_message.sql b/migrations/clickhouse/000008_message.sql index 6d48beb..b93a2f5 100644 --- a/migrations/clickhouse/000008_message.sql +++ b/migrations/clickhouse/000008_message.sql @@ -25,4 +25,5 @@ from ( arrayJoin(JSONExtractArrayRaw(JSONExtractString(tx, 'body', 'messages'))) AS msg, JSONExtractString(msg, '@type') AS type from spacebox.raw_transaction + where code = 0 ) \ No newline at end of file