From 2a905db21f2ffc44379662e14115aa98607395d1 Mon Sep 17 00:00:00 2001 From: Dario Russi <113150618+dariorussi@users.noreply.github.com> Date: Mon, 27 May 2024 22:01:19 +0200 Subject: [PATCH] indexer schema updates --- Cargo.lock | 2 + crates/sui-bridge-indexer/Cargo.toml | 18 ++++--- crates/sui-bridge-indexer/src/lib.rs | 12 +++-- .../up.sql | 18 +++++-- crates/sui-bridge-indexer/src/models.rs | 10 ++-- crates/sui-bridge-indexer/src/schema.rs | 18 ++++--- crates/sui-bridge-indexer/src/worker.rs | 53 ++++++++++++------- 7 files changed, 86 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 901015c02ad2d6..c4b691437fb3b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12096,6 +12096,8 @@ dependencies = [ "clap", "diesel", "ethers", + "fastcrypto", + "hex", "hex-literal 0.3.4", "mysten-metrics", "prometheus", diff --git a/crates/sui-bridge-indexer/Cargo.toml b/crates/sui-bridge-indexer/Cargo.toml index 66833b7044eead..64dbdbedd56279 100644 --- a/crates/sui-bridge-indexer/Cargo.toml +++ b/crates/sui-bridge-indexer/Cargo.toml @@ -11,18 +11,20 @@ serde.workspace = true diesel = { version = "2.1.4", features = ["postgres", "r2d2", "serde_json"] } ethers = "2.0" tokio = { workspace = true, features = ["full"] } -sui-types.workspace = true -prometheus.workspace = true +anyhow.workspace = true async-trait.workspace = true -sui-data-ingestion-core.workspace = true -sui-bridge.workspace = true -clap.workspace = true -tracing.workspace = true +bcs.workspace = true bin-version.workspace = true -anyhow.workspace = true +clap.workspace = true +fastcrypto.workspace = true +hex.workspace = true mysten-metrics.workspace = true -bcs.workspace = true +prometheus.workspace = true serde_yaml.workspace = true +sui-bridge.workspace = true +sui-data-ingestion-core.workspace = true +sui-types.workspace = true +tracing.workspace = true [dev-dependencies] sui-types = { workspace = true, features = ["test-utils"] } diff --git a/crates/sui-bridge-indexer/src/lib.rs b/crates/sui-bridge-indexer/src/lib.rs index 97bee58203d8ca..a31711fd2416f9 100644 --- a/crates/sui-bridge-indexer/src/lib.rs +++ b/crates/sui-bridge-indexer/src/lib.rs @@ -17,7 +17,8 @@ pub struct TokenTransfer { nonce: u64, block_height: u64, timestamp_ms: u64, - txn_hash: Vec, + txn_hash: String, + sender_address: String, status: TokenTransferStatus, gas_usage: i64, data_source: BridgeDataSource, @@ -25,9 +26,8 @@ pub struct TokenTransfer { } pub struct TokenTransferData { - sender_address: Vec, destination_chain: u8, - recipient_address: Vec, + recipient_address: String, token_id: u8, amount: u64, } @@ -40,6 +40,7 @@ impl From for DBTokenTransfer { block_height: value.block_height as i64, timestamp_ms: value.timestamp_ms as i64, txn_hash: value.txn_hash, + sender_address: value.sender_address.clone(), status: value.status.to_string(), gas_usage: value.gas_usage, data_source: value.data_source.to_string(), @@ -63,7 +64,10 @@ impl TryFrom<&TokenTransfer> for DBTokenTransferData { .map(|data| DBTokenTransferData { chain_id: value.chain_id as i32, nonce: value.nonce as i64, - sender_address: data.sender_address.clone(), + block_height: value.block_height as i64, + timestamp_ms: value.timestamp_ms as i64, + txn_hash: value.txn_hash.clone(), + sender_address: value.sender_address.clone(), destination_chain: data.destination_chain as i32, recipient_address: data.recipient_address.clone(), token_id: data.token_id as i32, diff --git a/crates/sui-bridge-indexer/src/migrations/00000000000000_diesel_initial_setup/up.sql b/crates/sui-bridge-indexer/src/migrations/00000000000000_diesel_initial_setup/up.sql index fffdaa5e6234b1..439051833efa99 100644 --- a/crates/sui-bridge-indexer/src/migrations/00000000000000_diesel_initial_setup/up.sql +++ b/crates/sui-bridge-indexer/src/migrations/00000000000000_diesel_initial_setup/up.sql @@ -2,13 +2,19 @@ CREATE TABLE token_transfer_data ( chain_id INT NOT NULL, nonce BIGINT NOT NULL, - sender_address bytea NOT NULL, + block_height BIGINT NOT NULL, + timestamp_ms BIGINT NOT NULL, + txn_hash TEXT NOT NULL, + sender_address TEXT NOT NULL, destination_chain INT NOT NULL, - recipient_address bytea NOT NULL, + recipient_address TEXT NOT NULL, token_id INT NOT NULL, amount BIGINT NOT NULL, PRIMARY KEY(chain_id, nonce) ); +CREATE INDEX token_transfer_data_block_height ON token_transfer_data (block_height); +CREATE INDEX token_transfer_data_timestamp_ms ON token_transfer_data (timestamp_ms); +CREATE INDEX token_transfer_data_sender_address ON token_transfer_data (sender_address); CREATE INDEX token_transfer_data_destination_chain ON token_transfer_data (destination_chain); CREATE INDEX token_transfer_data_token_id ON token_transfer_data (token_id); @@ -19,8 +25,12 @@ CREATE TABLE token_transfer status TEXT NOT NULL, block_height BIGINT NOT NULL, timestamp_ms BIGINT NOT NULL, - txn_hash bytea NOT NULL, + txn_hash TEXT NOT NULL, + sender_address TEXT NOT NULL, gas_usage BIGINT NOT NULL, data_source TEXT NOT NULL, PRIMARY KEY(chain_id, nonce, status) -); \ No newline at end of file +); +CREATE INDEX token_transfer_block_height ON token_transfer (block_height); +CREATE INDEX token_transfer_timestamp_ms ON token_transfer (timestamp_ms); +CREATE INDEX token_transfer_sender_address ON token_transfer (sender_address); diff --git a/crates/sui-bridge-indexer/src/models.rs b/crates/sui-bridge-indexer/src/models.rs index ee0bcc88ab6462..d729cf430327b0 100644 --- a/crates/sui-bridge-indexer/src/models.rs +++ b/crates/sui-bridge-indexer/src/models.rs @@ -12,7 +12,8 @@ pub struct TokenTransfer { pub status: String, pub block_height: i64, pub timestamp_ms: i64, - pub txn_hash: Vec, + pub txn_hash: String, + pub sender_address: String, pub gas_usage: i64, pub data_source: String, } @@ -22,9 +23,12 @@ pub struct TokenTransfer { pub struct TokenTransferData { pub chain_id: i32, pub nonce: i64, - pub sender_address: Vec, + pub block_height: i64, + pub timestamp_ms: i64, + pub txn_hash: String, + pub sender_address: String, pub destination_chain: i32, - pub recipient_address: Vec, + pub recipient_address: String, pub token_id: i32, pub amount: i64, } diff --git a/crates/sui-bridge-indexer/src/schema.rs b/crates/sui-bridge-indexer/src/schema.rs index e13e095c226544..baad2c2e600f15 100644 --- a/crates/sui-bridge-indexer/src/schema.rs +++ b/crates/sui-bridge-indexer/src/schema.rs @@ -1,6 +1,3 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - // @generated automatically by Diesel CLI. diesel::table! { @@ -10,7 +7,8 @@ diesel::table! { status -> Text, block_height -> Int8, timestamp_ms -> Int8, - txn_hash -> Bytea, + txn_hash -> Text, + sender_address -> Text, gas_usage -> Int8, data_source -> Text, } @@ -20,12 +18,18 @@ diesel::table! { token_transfer_data (chain_id, nonce) { chain_id -> Int4, nonce -> Int8, - sender_address -> Bytea, + block_height -> Int8, + timestamp_ms -> Int8, + txn_hash -> Text, + sender_address -> Text, destination_chain -> Int4, - recipient_address -> Bytea, + recipient_address -> Text, token_id -> Int4, amount -> Int8, } } -diesel::allow_tables_to_appear_in_same_query!(token_transfer, token_transfer_data,); +diesel::allow_tables_to_appear_in_same_query!( + token_transfer, + token_transfer_data, +); diff --git a/crates/sui-bridge-indexer/src/worker.rs b/crates/sui-bridge-indexer/src/worker.rs index b05bba87ffddad..2e35a601a433aa 100644 --- a/crates/sui-bridge-indexer/src/worker.rs +++ b/crates/sui-bridge-indexer/src/worker.rs @@ -8,6 +8,7 @@ use async_trait::async_trait; use ethers::providers::Provider; use ethers::providers::{Http, Middleware}; use ethers::types::Address as EthAddress; +use fastcrypto::encoding::{Base58, Encoding}; use std::collections::BTreeSet; use std::sync::Arc; use sui_bridge::abi::{EthBridgeEvent, EthSuiBridgeEvents}; @@ -16,9 +17,9 @@ use sui_bridge::events::{ }; use sui_bridge::types::EthLog; use sui_data_ingestion_core::Worker; -use sui_types::effects::TransactionEffectsAPI; use sui_types::{ base_types::ObjectID, + effects::TransactionEffectsAPI, full_checkpoint_content::{CheckpointData, CheckpointTransaction}, transaction::{TransactionDataAPI, TransactionKind}, BRIDGE_ADDRESS, SUI_BRIDGE_OBJECT_ID, @@ -57,30 +58,35 @@ impl BridgeWorker { // Process a transaction that has been identified as a bridge transaction. fn process_transaction(&self, tx: &CheckpointTransaction, checkpoint: u64, timestamp_ms: u64) { - if let Some(event) = &tx.events { - event.data.iter().for_each(|ev| { + if let Some(events) = &tx.events { + events.data.iter().for_each(|ev| { if ev.type_.address == BRIDGE_ADDRESS { let token_transfer = match ev.type_.name.as_str() { "TokenDepositedEvent" => { println!("Observed Sui Deposit"); - // todo: handle deserialization error - let event: MoveTokenDepositedEvent = + // TODO: handle deserialization error + let move_event: MoveTokenDepositedEvent = bcs::from_bytes(&ev.contents).unwrap(); Some(TokenTransfer { - chain_id: event.source_chain, - nonce: event.seq_num, + chain_id: move_event.source_chain, + nonce: move_event.seq_num, block_height: checkpoint, timestamp_ms, - txn_hash: tx.transaction.digest().inner().to_vec(), + txn_hash: tx.transaction.digest().base58_encode(), + // should this value come from ev.sender? + // sender_address: move_event.sender_address, + sender_address: ev.sender.to_string(), status: TokenTransferStatus::Deposited, gas_usage: tx.effects.gas_cost_summary().net_gas_usage(), data_source: BridgeDataSource::Sui, data: Some(TokenTransferData { - sender_address: event.sender_address, - destination_chain: event.target_chain, - recipient_address: event.target_address, - token_id: event.token_type, - amount: event.amount_sui_adjusted, + destination_chain: move_event.target_chain, + recipient_address: format!( + "0x{}", + hex::encode(move_event.target_address), + ), + token_id: move_event.token_type, + amount: move_event.amount_sui_adjusted, }), }) } @@ -93,7 +99,8 @@ impl BridgeWorker { nonce: event.message_key.bridge_seq_num, block_height: checkpoint, timestamp_ms, - txn_hash: tx.transaction.digest().inner().to_vec(), + txn_hash: tx.transaction.digest().base58_encode(), + sender_address: ev.sender.to_string(), status: TokenTransferStatus::Approved, gas_usage: tx.effects.gas_cost_summary().net_gas_usage(), data_source: BridgeDataSource::Sui, @@ -109,7 +116,8 @@ impl BridgeWorker { nonce: event.message_key.bridge_seq_num, block_height: checkpoint, timestamp_ms, - txn_hash: tx.transaction.digest().inner().to_vec(), + txn_hash: tx.transaction.digest().base58_encode(), + sender_address: ev.sender.to_string(), status: TokenTransferStatus::Claimed, gas_usage: tx.effects.gas_cost_summary().net_gas_usage(), data_source: BridgeDataSource::Sui, @@ -163,14 +171,20 @@ pub async fn process_eth_transaction( nonce: bridge_event.nonce, block_height: block_number, timestamp_ms: timestamp, - txn_hash: tx_hash.as_bytes().to_vec(), + txn_hash: Base58::encode(tx_hash.as_bytes()), + sender_address: format!( + "0x{}", + hex::encode(bridge_event.sender_address.as_bytes()), + ), status: TokenTransferStatus::Deposited, gas_usage: gas.as_u64() as i64, data_source: BridgeDataSource::Eth, data: Some(TokenTransferData { - sender_address: bridge_event.sender_address.as_bytes().to_vec(), destination_chain: bridge_event.destination_chain_id, - recipient_address: bridge_event.recipient_address.to_vec(), + recipient_address: format!( + "0x{}", + hex::encode(bridge_event.recipient_address), + ), token_id: bridge_event.token_id, amount: bridge_event.sui_adjusted_amount, }), @@ -185,7 +199,8 @@ pub async fn process_eth_transaction( nonce: bridge_event.nonce, block_height: block_number, timestamp_ms: timestamp, - txn_hash: tx_hash.as_bytes().to_vec(), + txn_hash: Base58::encode(tx_hash.as_bytes()), + sender_address: bridge_event.sender_address.to_string(), status: TokenTransferStatus::Claimed, gas_usage: gas.as_u64() as i64, data_source: BridgeDataSource::Eth,