From 89e0159b5cb5e8337ba4d6702c7925f3fb15df28 Mon Sep 17 00:00:00 2001 From: Renee Tso <8248583+rtso@users.noreply.github.com> Date: Fri, 31 Jan 2025 22:19:50 -0500 Subject: [PATCH 1/4] Refactor events models (#699) --- rust/Cargo.lock | 6 +- rust/Cargo.toml | 2 +- .../validator_txn_missing_events/events.json | 1 + .../src/models/events_models.rs | 3 +- .../src/sdk_tests/events_processor_tests.rs | 18 ++ .../common/models/event_models/raw_events.rs | 72 ++++++- .../default_models/parquet_transactions.rs | 18 +- .../models/event_models/parquet_events.rs | 195 ++---------------- .../postgres/models/events_models/events.rs | 73 ++----- .../src/processors/events_processor.rs | 49 ++--- .../parquet_events_processor.rs | 64 ++---- rust/processor/src/utils/util.rs | 10 +- .../src/config/processor_config.rs | 2 +- .../db/common/models/events_models/events.rs | 79 ------- .../src/db/common/models/events_models/mod.rs | 4 - .../sdk-processor/src/db/common/models/mod.rs | 1 - .../src/parquet_processors/mod.rs | 6 +- .../parquet_events_processor.rs | 2 +- .../events_processor/events_extractor.rs | 57 +---- .../steps/events_processor/events_storer.rs | 17 +- 20 files changed, 186 insertions(+), 493 deletions(-) create mode 100644 rust/integration-tests/sdk_expected_db_output_files/imported_mainnet_txns/events_processor/validator_txn_missing_events/events.json delete mode 100644 rust/sdk-processor/src/db/common/models/events_models/events.rs delete mode 100644 rust/sdk-processor/src/db/common/models/events_models/mod.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 981519ed8..f5ea5c716 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "aptos-indexer-test-transactions" version = "1.0.0" -source = "git+https://github.com/aptos-labs/aptos-core.git?rev=63eb8945a86521bc0fb0e26b306693e11327c85d#63eb8945a86521bc0fb0e26b306693e11327c85d" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=b91d6bfe955e0df12927ae06f44c9763f5f016d3#b91d6bfe955e0df12927ae06f44c9763f5f016d3" dependencies = [ - "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=63eb8945a86521bc0fb0e26b306693e11327c85d)", + "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=b91d6bfe955e0df12927ae06f44c9763f5f016d3)", "serde_json", ] @@ -309,7 +309,7 @@ dependencies = [ [[package]] name = "aptos-protos" version = "1.3.1" -source = "git+https://github.com/aptos-labs/aptos-core.git?rev=63eb8945a86521bc0fb0e26b306693e11327c85d#63eb8945a86521bc0fb0e26b306693e11327c85d" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=b91d6bfe955e0df12927ae06f44c9763f5f016d3#b91d6bfe955e0df12927ae06f44c9763f5f016d3" dependencies = [ "pbjson", "prost 0.13.4", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index afb74f120..b5d51df30 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,7 +31,7 @@ anyhow = "1.0.86" aptos-indexer-processor-sdk = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "b79ed8b5864b2a12a1f9c5fd01579462e029b2ae" } aptos-indexer-processor-sdk-server-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "b79ed8b5864b2a12a1f9c5fd01579462e029b2ae" } aptos-protos = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "1d8460a995503574ec4e9699d3442d0150d7f3b9" } -aptos-indexer-test-transactions = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "63eb8945a86521bc0fb0e26b306693e11327c85d" } +aptos-indexer-test-transactions = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "b91d6bfe955e0df12927ae06f44c9763f5f016d3" } aptos-indexer-testing-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "b79ed8b5864b2a12a1f9c5fd01579462e029b2ae" } async-trait = "0.1.53" backtrace = "0.3.58" diff --git a/rust/integration-tests/sdk_expected_db_output_files/imported_mainnet_txns/events_processor/validator_txn_missing_events/events.json b/rust/integration-tests/sdk_expected_db_output_files/imported_mainnet_txns/events_processor/validator_txn_missing_events/events.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/imported_mainnet_txns/events_processor/validator_txn_missing_events/events.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/src/models/events_models.rs b/rust/integration-tests/src/models/events_models.rs index 1a7e05f99..3255dd3ee 100644 --- a/rust/integration-tests/src/models/events_models.rs +++ b/rust/integration-tests/src/models/events_models.rs @@ -10,8 +10,7 @@ use serde::{Deserialize, Serialize}; /** * Event model * this is created // b/c there is inserated_at field which isn't defined in the Event struct, we can't just load the events directly without specifying the fields. - // TODO: make this more generic to load all fields, then we should be able to run tests for all processor in one test case. - +* TODO: make this more generic to load all fields, then we should be able to run tests for all processor in one test case. */ #[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize, Queryable)] #[diesel(primary_key(transaction_version, event_index))] diff --git a/rust/integration-tests/src/sdk_tests/events_processor_tests.rs b/rust/integration-tests/src/sdk_tests/events_processor_tests.rs index 62be9b180..10f4e9bc1 100644 --- a/rust/integration-tests/src/sdk_tests/events_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/events_processor_tests.rs @@ -52,6 +52,7 @@ mod tests { IMPORTED_DEVNET_TXNS_78753811_COIN_TRANSFER_WITH_V2_EVENTS, IMPORTED_DEVNET_TXNS_78753831_TOKEN_V1_MINT_TRANSFER_WITH_V2_EVENTS, IMPORTED_DEVNET_TXNS_78753832_TOKEN_V2_MINT_TRANSFER_WITH_V2_EVENTS, + IMPORTED_MAINNET_TXNS_554229017_EVENTS_WITH_NO_EVENT_SIZE_INFO, IMPORTED_TESTNET_TXNS_1255836496_V2_FA_METADATA_, IMPORTED_TESTNET_TXNS_1_GENESIS, IMPORTED_TESTNET_TXNS_278556781_V1_COIN_REGISTER_FA_METADATA, IMPORTED_TESTNET_TXNS_2_NEW_BLOCK_EVENT, IMPORTED_TESTNET_TXNS_3_EMPTY_TXN, @@ -151,6 +152,19 @@ mod tests { .await; } + // This is a test for the validator txn with missing events + // This happens because we did not fully backfill validator txn events + // so GRPC can return a txn with event size info but no events + // We expect no events parsed + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn mainnet_events_processor_validator_txn_missing_events() { + process_single_mainnet_event_txn( + IMPORTED_MAINNET_TXNS_554229017_EVENTS_WITH_NO_EVENT_SIZE_INFO, // this is misnamed, but it's the validatortxn with missing events + Some("validator_txn_missing_events".to_string()), + ) + .await; + } + // Example 2: Test for multiple transactions handling #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn testnet_events_processor_db_output_scenario_testing() { @@ -218,6 +232,10 @@ mod tests { process_single_event_txn(txn, test_case_name, "imported_testnet_txns").await } + async fn process_single_mainnet_event_txn(txn: &[u8], test_case_name: Option) { + process_single_event_txn(txn, test_case_name, "imported_mainnet_txns").await + } + // Helper function to abstract out the single transaction processing async fn process_single_event_txn( txn: &[u8], diff --git a/rust/processor/src/db/common/models/event_models/raw_events.rs b/rust/processor/src/db/common/models/event_models/raw_events.rs index 9ef0fdf30..bf72789c0 100644 --- a/rust/processor/src/db/common/models/event_models/raw_events.rs +++ b/rust/processor/src/db/common/models/event_models/raw_events.rs @@ -1,7 +1,12 @@ -use crate::utils::util::{standardize_address, truncate_str}; -use aptos_protos::transaction::v1::{Event as EventPB, EventSizeInfo}; +use crate::utils::{ + counters::PROCESSOR_UNKNOWN_TYPE_COUNT, + util::{parse_timestamp, standardize_address, truncate_str}, +}; +use aptos_protos::transaction::v1::{ + transaction::TxnData, Event as EventPB, EventSizeInfo, Transaction, +}; use serde::{Deserialize, Serialize}; - +use tracing::warn; /// P99 currently is 303 so using 300 as a safe max length pub const EVENT_TYPE_MAX_LENGTH: usize = 300; @@ -21,12 +26,8 @@ pub struct RawEvent { pub total_bytes: Option, } -pub trait EventConvertible { - fn from_raw(raw_item: &RawEvent) -> Self; -} - impl RawEvent { - pub fn from_raw_event( + fn from_event( event: &EventPB, txn_version: i64, txn_block_height: i64, @@ -56,3 +57,58 @@ impl RawEvent { } } } + +pub fn parse_events(txn: &Transaction, processor_name: &str) -> Vec { + let txn_version = txn.version as i64; + let block_height = txn.block_height as i64; + let block_timestamp = parse_timestamp(txn.timestamp.as_ref().unwrap(), txn_version); + let size_info = match txn.size_info.as_ref() { + Some(size_info) => Some(size_info), + None => { + warn!(version = txn.version, "Transaction size info not found"); + None + }, + }; + let txn_data = match txn.txn_data.as_ref() { + Some(data) => data, + None => { + warn!( + transaction_version = txn_version, + "Transaction data doesn't exist" + ); + PROCESSOR_UNKNOWN_TYPE_COUNT + .with_label_values(&[processor_name]) + .inc(); + return vec![]; + }, + }; + let default = vec![]; + let raw_events = match txn_data { + TxnData::BlockMetadata(tx_inner) => &tx_inner.events, + TxnData::Genesis(tx_inner) => &tx_inner.events, + TxnData::User(tx_inner) => &tx_inner.events, + TxnData::Validator(tx_inner) => &tx_inner.events, + _ => &default, + }; + + let event_size_info = size_info.map(|info| info.event_size_info.as_slice()); + + raw_events + .iter() + .enumerate() + .map(|(index, event)| { + // event_size_info will be used for user transactions only, no promises for other transactions. + // If event_size_info is missing due, it defaults to 0. + // No need to backfill as event_size_info is primarily for debugging user transactions. + let size_info = event_size_info.and_then(|infos| infos.get(index)); + RawEvent::from_event( + event, + txn_version, + block_height, + index as i64, + size_info, + Some(block_timestamp), + ) + }) + .collect::>() +} diff --git a/rust/processor/src/db/parquet/models/default_models/parquet_transactions.rs b/rust/processor/src/db/parquet/models/default_models/parquet_transactions.rs index 403463284..2e22e1a18 100644 --- a/rust/processor/src/db/parquet/models/default_models/parquet_transactions.rs +++ b/rust/processor/src/db/parquet/models/default_models/parquet_transactions.rs @@ -183,6 +183,8 @@ impl Transaction { .expect("Txn Timestamp is invalid!"); let txn_size_info = transaction.size_info.as_ref(); + let len_event_size_info = + txn_size_info.map(|size_info| size_info.event_size_info.len() as i64); match txn_data { TxnData::User(user_txn) => { @@ -204,6 +206,10 @@ impl Transaction { }, None => (None, None), }; + // For some older validator transactions, GRPC stubs out the raw events. + // We use the event size info as the source of truth to determine the number of events in the transaction. + // This inconsistency should only be temporary until all transaction events are backfilled in GRPC. + let num_events = len_event_size_info.unwrap_or(user_txn.events.len() as i64); let serialized_payload = payload_cleaned.map(|payload| canonical_json::to_string(&payload).unwrap()); @@ -214,7 +220,7 @@ impl Transaction { payload_type, txn_version, transaction_type, - user_txn.events.len() as i64, + num_events, block_height, epoch, block_timestamp, @@ -239,6 +245,7 @@ impl Transaction { payload_cleaned.map(|payload| canonical_json::to_string(&payload).unwrap()); let payload_type = None; + let num_events = len_event_size_info.unwrap_or(genesis_txn.events.len() as i64); ( Self::from_transaction_info_with_data( transaction_info, @@ -246,7 +253,7 @@ impl Transaction { payload_type, txn_version, transaction_type, - genesis_txn.events.len() as i64, + num_events, block_height, epoch, block_timestamp, @@ -263,6 +270,8 @@ impl Transaction { block_height, block_timestamp, ); + let num_events = + len_event_size_info.unwrap_or(block_metadata_txn.events.len() as i64); ( Self::from_transaction_info_with_data( transaction_info, @@ -270,7 +279,7 @@ impl Transaction { None, txn_version, transaction_type, - block_metadata_txn.events.len() as i64, + num_events, block_height, epoch, block_timestamp, @@ -303,6 +312,7 @@ impl Transaction { block_height, block_timestamp, ); + let num_events = len_event_size_info.unwrap_or(inner.events.len() as i64); ( Self::from_transaction_info_with_data( transaction_info, @@ -310,7 +320,7 @@ impl Transaction { None, txn_version, transaction_type, - inner.events.len() as i64, + num_events, block_height, epoch, block_timestamp, diff --git a/rust/processor/src/db/parquet/models/event_models/parquet_events.rs b/rust/processor/src/db/parquet/models/event_models/parquet_events.rs index 14953890f..de97aec5d 100644 --- a/rust/processor/src/db/parquet/models/event_models/parquet_events.rs +++ b/rust/processor/src/db/parquet/models/event_models/parquet_events.rs @@ -4,30 +4,14 @@ #![allow(clippy::extra_unused_lifetimes)] use crate::{ bq_analytics::generic_parquet_processor::{GetTimeStamp, HasVersion, NamedTable}, - db::common::models::event_models::raw_events::{ - EventConvertible, RawEvent, EVENT_TYPE_MAX_LENGTH, - }, - utils::util::truncate_str, + db::common::models::event_models::raw_events::RawEvent, }; use allocative_derive::Allocative; -use aptos_protos::transaction::v1::{Event as EventPB, EventSizeInfo}; -use lazy_static::lazy_static; use parquet_derive::ParquetRecordWriter; use serde::{Deserialize, Serialize}; -const DEFAULT_CREATION_NUMBER: i64 = 0; -const DEFAULT_SEQUENCE_NUMBER: i64 = 0; -// This is for future proofing. TODO: change when events v2 comes -const EVENT_VERSION: i8 = 1i8; - -lazy_static! { - pub static ref DEFAULT_ACCOUNT_ADDRESS: String = "NULL_ACCOUNT_ADDRESS".to_string(); - pub static ref DEFAULT_EVENT_TYPE: String = "NULL_EVENT_TYPE".to_string(); - pub static ref DEFAULT_EVENT_DATA: String = "NULL_EVENT_DATA".to_string(); -} - #[derive(Allocative, Clone, Debug, Default, Deserialize, ParquetRecordWriter, Serialize)] -pub struct Event { +pub struct EventPQ { pub txn_version: i64, pub account_address: String, pub sequence_number: i64, @@ -39,180 +23,41 @@ pub struct Event { pub indexed_type: String, pub type_tag_bytes: i64, pub total_bytes: i64, - pub event_version: i8, #[allocative(skip)] pub block_timestamp: chrono::NaiveDateTime, } -impl NamedTable for Event { +impl NamedTable for EventPQ { const TABLE_NAME: &'static str = "events"; } -impl HasVersion for Event { +impl HasVersion for EventPQ { fn version(&self) -> i64 { self.txn_version } } -impl GetTimeStamp for Event { +impl GetTimeStamp for EventPQ { fn get_timestamp(&self) -> chrono::NaiveDateTime { self.block_timestamp } } -impl Event { - pub fn from_event( - event: &EventPB, - txn_version: i64, - block_height: i64, - event_index: i64, - size_info: &EventSizeInfo, - block_timestamp: chrono::NaiveDateTime, - ) -> Self { - let raw = RawEvent::from_raw_event( - event, - txn_version, - block_height, - event_index, - Some(size_info), - Some(block_timestamp), - ); - Self::from_raw(&raw) - } - - // This function is added to handle the txn with events filtered, but event_size_info is not filtered. - pub fn from_null_event( - txn_version: i64, - block_height: i64, - event_index: i64, - size_info: &EventSizeInfo, - block_timestamp: chrono::NaiveDateTime, - ) -> Self { - Event { - account_address: DEFAULT_ACCOUNT_ADDRESS.clone(), - creation_number: DEFAULT_CREATION_NUMBER, - sequence_number: DEFAULT_SEQUENCE_NUMBER, - txn_version, - block_height, - event_type: DEFAULT_EVENT_TYPE.clone(), - data: DEFAULT_EVENT_DATA.clone(), - event_index, - indexed_type: truncate_str(&DEFAULT_EVENT_TYPE, EVENT_TYPE_MAX_LENGTH), - type_tag_bytes: size_info.type_tag_bytes as i64, - total_bytes: size_info.total_bytes as i64, - event_version: EVENT_VERSION, - block_timestamp, - } - } - - pub fn from_events( - events: &[EventPB], - txn_version: i64, - block_height: i64, - event_size_info: &[EventSizeInfo], - block_timestamp: chrono::NaiveDateTime, - is_user_txn_type: bool, - ) -> Vec { - let mut temp_events = events.to_vec(); - - // event_size_info will be used for user transactions only, no promises for other transactions. - // If event_size_info is missing due to fewer or no events, it defaults to 0. - // No need to backfill as event_size_info is primarily for debugging user transactions. - if temp_events.len() != event_size_info.len() { - tracing::warn!( - events_len = events.len(), - event_size_info_len = event_size_info.len(), - txn_version, - "Length mismatch: events size does not match event_size_info size.", - ); - if is_user_txn_type { - return handle_user_txn_type( - &mut temp_events, - txn_version, - event_size_info, - block_timestamp, - block_height, - ); - } +impl From for EventPQ { + fn from(raw_event: RawEvent) -> Self { + EventPQ { + txn_version: raw_event.transaction_version, + account_address: raw_event.account_address, + sequence_number: raw_event.sequence_number, + creation_number: raw_event.creation_number, + block_height: raw_event.transaction_block_height, + event_type: raw_event.type_, + data: raw_event.data, + event_index: raw_event.event_index, + indexed_type: raw_event.indexed_type, + type_tag_bytes: raw_event.type_tag_bytes.unwrap_or(0), + total_bytes: raw_event.total_bytes.unwrap_or(0), + block_timestamp: raw_event.block_timestamp.unwrap(), } - temp_events - .iter() - .enumerate() - .map(|(index, event)| { - let size_info = event_size_info.get(index).unwrap_or(&EventSizeInfo { - type_tag_bytes: 0, - total_bytes: 0, - }); - Self::from_event( - event, - txn_version, - block_height, - index as i64, - size_info, - block_timestamp, - ) - }) - .collect::>() } } - -fn handle_user_txn_type( - temp_events: &mut Vec, - txn_version: i64, - event_size_info: &[EventSizeInfo], - block_timestamp: chrono::NaiveDateTime, - block_height: i64, -) -> Vec { - if event_size_info.is_empty() { - tracing::error!( - txn_version, - "Event size info is missing for user transaction." - ); - panic!("Event size info is missing for user transaction."); - } - // Add default events to temp_events until its length matches event_size_info length - tracing::info!( - txn_version, - "Events are empty but event_size_info is not empty." - ); - temp_events.resize(event_size_info.len(), EventPB::default()); - temp_events - .iter() - .enumerate() - .map(|(index, _event)| { - let size_info = event_size_info.get(index).unwrap_or(&EventSizeInfo { - type_tag_bytes: 0, - total_bytes: 0, - }); - Event::from_null_event( - txn_version, - block_height, - index as i64, - size_info, - block_timestamp, - ) - }) - .collect() -} - -impl EventConvertible for Event { - fn from_raw(raw: &RawEvent) -> Self { - Event { - txn_version: raw.transaction_version, - account_address: raw.account_address.clone(), - sequence_number: raw.sequence_number, - creation_number: raw.creation_number, - block_height: raw.transaction_block_height, - event_type: raw.type_.clone(), - data: raw.data.clone(), - event_index: raw.event_index, - indexed_type: raw.indexed_type.clone(), - type_tag_bytes: raw.type_tag_bytes.unwrap_or(0), - total_bytes: raw.total_bytes.unwrap_or(0), - event_version: EVENT_VERSION, - block_timestamp: raw.block_timestamp.unwrap(), - } - } -} - -pub type ParquetEventModel = Event; diff --git a/rust/processor/src/db/postgres/models/events_models/events.rs b/rust/processor/src/db/postgres/models/events_models/events.rs index d1e010484..b4f1a53d6 100644 --- a/rust/processor/src/db/postgres/models/events_models/events.rs +++ b/rust/processor/src/db/postgres/models/events_models/events.rs @@ -3,18 +3,14 @@ #![allow(clippy::extra_unused_lifetimes)] -use crate::{ - db::common::models::event_models::raw_events::{EventConvertible, RawEvent}, - schema::events, -}; -use aptos_protos::transaction::v1::Event as EventPB; +use crate::{db::common::models::event_models::raw_events::RawEvent, schema::events}; use field_count::FieldCount; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)] #[diesel(primary_key(transaction_version, event_index))] #[diesel(table_name = events)] -pub struct Event { +pub struct EventPG { pub sequence_number: i64, pub creation_number: i64, pub account_address: String, @@ -26,59 +22,18 @@ pub struct Event { pub indexed_type: String, } -impl Event { - pub fn from_event( - event: &EventPB, - transaction_version: i64, - transaction_block_height: i64, - event_index: i64, - ) -> Self { - let raw = RawEvent::from_raw_event( - event, - transaction_version, - transaction_block_height, - event_index, - None, - None, - ); - Self::from_raw(&raw) - } - - pub fn from_events( - events: &[EventPB], - transaction_version: i64, - transaction_block_height: i64, - ) -> Vec { - events - .iter() - .enumerate() - .map(|(index, event)| { - Self::from_event( - event, - transaction_version, - transaction_block_height, - index as i64, - ) - }) - .collect::>() - } -} - -impl EventConvertible for Event { - fn from_raw(raw: &RawEvent) -> Self { - Event { - sequence_number: raw.sequence_number, - creation_number: raw.creation_number, - account_address: raw.account_address.clone(), - transaction_version: raw.transaction_version, - transaction_block_height: raw.transaction_block_height, - type_: raw.type_.clone(), - data: serde_json::from_str(&raw.data).unwrap(), - event_index: raw.event_index, - indexed_type: raw.indexed_type.clone(), +impl From for EventPG { + fn from(raw_event: RawEvent) -> Self { + EventPG { + sequence_number: raw_event.sequence_number, + creation_number: raw_event.creation_number, + account_address: raw_event.account_address, + transaction_version: raw_event.transaction_version, + transaction_block_height: raw_event.transaction_block_height, + type_: raw_event.type_, + data: serde_json::from_str(&raw_event.data).unwrap(), + event_index: raw_event.event_index, + indexed_type: raw_event.indexed_type, } } } - -// Prevent conflicts with other things named `Event` -pub type EventModel = Event; diff --git a/rust/processor/src/processors/events_processor.rs b/rust/processor/src/processors/events_processor.rs index 0154b9f0e..ba48b95a8 100644 --- a/rust/processor/src/processors/events_processor.rs +++ b/rust/processor/src/processors/events_processor.rs @@ -3,17 +3,17 @@ use super::{DefaultProcessingResult, ProcessorName, ProcessorTrait}; use crate::{ - db::postgres::models::events_models::events::EventModel, + db::{ + common::models::event_models::raw_events::parse_events, + postgres::models::events_models::events::EventPG, + }, gap_detectors::ProcessingResult, schema, - utils::{ - counters::PROCESSOR_UNKNOWN_TYPE_COUNT, - database::{execute_in_chunks, get_config_table_chunk_size, ArcDbPool}, - }, + utils::database::{execute_in_chunks, get_config_table_chunk_size, ArcDbPool}, }; use ahash::AHashMap; use anyhow::bail; -use aptos_protos::transaction::v1::{transaction::TxnData, Transaction}; +use aptos_protos::transaction::v1::Transaction; use async_trait::async_trait; use diesel::{ pg::{upsert::excluded, Pg}, @@ -53,7 +53,7 @@ async fn insert_to_db( name: &'static str, start_version: u64, end_version: u64, - events: &[EventModel], + events: &[EventPG], per_table_chunk_sizes: &AHashMap, ) -> Result<(), diesel::result::Error> { tracing::trace!( @@ -66,14 +66,14 @@ async fn insert_to_db( conn, insert_events_query, events, - get_config_table_chunk_size::("events", per_table_chunk_sizes), + get_config_table_chunk_size::("events", per_table_chunk_sizes), ) .await?; Ok(()) } fn insert_events_query( - items_to_insert: Vec, + items_to_insert: Vec, ) -> ( impl QueryFragment + diesel::query_builder::QueryId + Send, Option<&'static str>, @@ -152,34 +152,13 @@ impl ProcessorTrait for EventsProcessor { } } -pub fn process_transactions(transactions: Vec) -> Vec { +pub fn process_transactions(transactions: Vec) -> Vec { let mut events = vec![]; for txn in &transactions { - let txn_version = txn.version as i64; - let block_height = txn.block_height as i64; - let txn_data = match txn.txn_data.as_ref() { - Some(data) => data, - None => { - tracing::warn!( - transaction_version = txn_version, - "Transaction data doesn't exist" - ); - PROCESSOR_UNKNOWN_TYPE_COUNT - .with_label_values(&["EventsProcessor"]) - .inc(); - continue; - }, - }; - let default = vec![]; - let raw_events = match txn_data { - TxnData::BlockMetadata(tx_inner) => &tx_inner.events, - TxnData::Genesis(tx_inner) => &tx_inner.events, - TxnData::User(tx_inner) => &tx_inner.events, - TxnData::Validator(tx_inner) => &tx_inner.events, - _ => &default, - }; - - let txn_events = EventModel::from_events(raw_events, txn_version, block_height); + let txn_events: Vec = parse_events(txn, "EventsProcessor") + .into_iter() + .map(|e| e.into()) + .collect(); events.extend(txn_events); } events diff --git a/rust/processor/src/processors/parquet_processors/parquet_events_processor.rs b/rust/processor/src/processors/parquet_processors/parquet_events_processor.rs index 9ef32206a..bfdce0a2f 100644 --- a/rust/processor/src/processors/parquet_processors/parquet_events_processor.rs +++ b/rust/processor/src/processors/parquet_processors/parquet_events_processor.rs @@ -6,19 +6,21 @@ use crate::{ create_parquet_handler_loop, generic_parquet_processor::ParquetDataGeneric, ParquetProcessingResult, }, - db::parquet::models::event_models::parquet_events::{Event, ParquetEventModel}, + db::{ + common::models::event_models::raw_events::parse_events, + parquet::models::event_models::parquet_events::EventPQ, + }, gap_detectors::ProcessingResult, processors::{parquet_processors::ParquetProcessorTrait, ProcessorName, ProcessorTrait}, - utils::{counters::PROCESSOR_UNKNOWN_TYPE_COUNT, database::ArcDbPool, util::parse_timestamp}, + utils::database::ArcDbPool, }; use ahash::AHashMap; use anyhow::Context; -use aptos_protos::transaction::v1::{transaction::TxnData, Transaction}; +use aptos_protos::transaction::v1::Transaction; use async_trait::async_trait; use kanal::AsyncSender; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, time::Duration}; -use tracing::warn; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct ParquetEventsProcessorConfig { @@ -38,7 +40,7 @@ impl ParquetProcessorTrait for ParquetEventsProcessorConfig { pub struct ParquetEventsProcessor { connection_pool: ArcDbPool, - event_sender: AsyncSender>, + event_sender: AsyncSender>, } impl ParquetEventsProcessor { @@ -49,7 +51,7 @@ impl ParquetEventsProcessor { ) -> Self { config.set_google_credentials(config.google_application_credentials.clone()); - let event_sender = create_parquet_handler_loop::( + let event_sender = create_parquet_handler_loop::( new_gap_detector_sender.clone(), ProcessorName::ParquetDefaultProcessor.into(), config.bucket_name.clone(), @@ -120,56 +122,16 @@ impl ProcessorTrait for ParquetEventsProcessor { pub fn process_transactions_parquet( transactions: Vec, -) -> (AHashMap, Vec) { +) -> (AHashMap, Vec) { let mut transaction_version_to_struct_count: AHashMap = AHashMap::new(); let mut events = vec![]; for txn in &transactions { let txn_version = txn.version as i64; - let block_height = txn.block_height as i64; - let block_timestamp = parse_timestamp(txn.timestamp.as_ref().unwrap(), txn_version); - let size_info = match txn.size_info.as_ref() { - Some(size_info) => size_info, - None => { - warn!(version = txn.version, "Transaction size info not found"); - continue; - }, - }; - let txn_data = match txn.txn_data.as_ref() { - Some(data) => data, - None => { - tracing::warn!( - transaction_version = txn_version, - "Transaction data doesn't exist" - ); - PROCESSOR_UNKNOWN_TYPE_COUNT - .with_label_values(&["ParquetEventsProcessor"]) - .inc(); - - continue; - }, - }; - let default = vec![]; - let mut is_user_txn_type = false; - let raw_events = match txn_data { - TxnData::BlockMetadata(tx_inner) => &tx_inner.events, - TxnData::Genesis(tx_inner) => &tx_inner.events, - TxnData::User(tx_inner) => { - is_user_txn_type = true; - &tx_inner.events - }, - TxnData::Validator(txn) => &txn.events, - _ => &default, - }; - - let txn_events = ParquetEventModel::from_events( - raw_events, - txn_version, - block_height, - size_info.event_size_info.as_slice(), - block_timestamp, - is_user_txn_type, - ); + let txn_events: Vec = parse_events(txn, "ParquetEventsProcessor") + .into_iter() + .map(|e| e.into()) + .collect(); transaction_version_to_struct_count .entry(txn_version) .and_modify(|e| *e += txn_events.len() as i64) diff --git a/rust/processor/src/utils/util.rs b/rust/processor/src/utils/util.rs index 51b748d01..dd979bf92 100644 --- a/rust/processor/src/utils/util.rs +++ b/rust/processor/src/utils/util.rs @@ -470,20 +470,14 @@ pub fn convert_bcs_hex_new(typ: u8, value: String) -> Option { /// Convert the json serialized PropertyMap's inner BCS fields to their original value in string format pub fn convert_bcs_propertymap(s: Value) -> Option { match PropertyMap::from_bcs_encode_str(s) { - Some(e) => match serde_json::to_value(&e) { - Ok(val) => Some(val), - Err(_) => None, - }, + Some(e) => serde_json::to_value(&e).ok(), None => None, } } pub fn convert_bcs_token_object_propertymap(s: Value) -> Option { match TokenObjectPropertyMap::from_bcs_encode_str(s) { - Some(e) => match serde_json::to_value(&e) { - Ok(val) => Some(val), - Err(_) => None, - }, + Some(e) => serde_json::to_value(&e).ok(), None => None, } } diff --git a/rust/sdk-processor/src/config/processor_config.rs b/rust/sdk-processor/src/config/processor_config.rs index 445f351bf..b692e8540 100644 --- a/rust/sdk-processor/src/config/processor_config.rs +++ b/rust/sdk-processor/src/config/processor_config.rs @@ -23,7 +23,7 @@ use processor::{ parquet_transactions::Transaction, parquet_write_set_changes::WriteSetChangeModel, }, - event_models::parquet_events::Event as EventPQ, + event_models::parquet_events::EventPQ, fungible_asset_models::{ parquet_v2_fungible_asset_activities::FungibleAssetActivity, parquet_v2_fungible_asset_balances::{ diff --git a/rust/sdk-processor/src/db/common/models/events_models/events.rs b/rust/sdk-processor/src/db/common/models/events_models/events.rs deleted file mode 100644 index fa7699d50..000000000 --- a/rust/sdk-processor/src/db/common/models/events_models/events.rs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -#![allow(clippy::extra_unused_lifetimes)] - -// Copied from processor crate. The only difference is the protos are imported from the SDK -// instead of the aptos-protos crate. -use aptos_indexer_processor_sdk::aptos_protos::transaction::v1::Event as EventPB; -use diesel::{Identifiable, Insertable}; -use field_count::FieldCount; -use processor::{ - schema::events, - utils::util::{standardize_address, truncate_str}, -}; -use serde::{Deserialize, Serialize}; - -// p99 currently is 303 so using 300 as a safe max length -const EVENT_TYPE_MAX_LENGTH: usize = 300; - -#[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)] -#[diesel(primary_key(transaction_version, event_index))] -#[diesel(table_name = events)] -pub struct Event { - pub sequence_number: i64, - pub creation_number: i64, - pub account_address: String, - pub transaction_version: i64, - pub transaction_block_height: i64, - pub type_: String, - pub data: serde_json::Value, - pub event_index: i64, - pub indexed_type: String, -} - -impl Event { - pub fn from_event( - event: &EventPB, - transaction_version: i64, - transaction_block_height: i64, - event_index: i64, - ) -> Self { - let t: &str = event.type_str.as_ref(); - Event { - account_address: standardize_address( - event.key.as_ref().unwrap().account_address.as_str(), - ), - creation_number: event.key.as_ref().unwrap().creation_number as i64, - sequence_number: event.sequence_number as i64, - transaction_version, - transaction_block_height, - type_: t.to_string(), - data: serde_json::from_str(event.data.as_str()).unwrap(), - event_index, - indexed_type: truncate_str(t, EVENT_TYPE_MAX_LENGTH), - } - } - - pub fn from_events( - events: &[EventPB], - transaction_version: i64, - transaction_block_height: i64, - ) -> Vec { - events - .iter() - .enumerate() - .map(|(index, event)| { - Self::from_event( - event, - transaction_version, - transaction_block_height, - index as i64, - ) - }) - .collect::>() - } -} - -// Prevent conflicts with other things named `Event` -pub type EventModel = Event; diff --git a/rust/sdk-processor/src/db/common/models/events_models/mod.rs b/rust/sdk-processor/src/db/common/models/events_models/mod.rs deleted file mode 100644 index 9d363699e..000000000 --- a/rust/sdk-processor/src/db/common/models/events_models/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -pub mod events; diff --git a/rust/sdk-processor/src/db/common/models/mod.rs b/rust/sdk-processor/src/db/common/models/mod.rs index 5393144c3..36bb119e9 100644 --- a/rust/sdk-processor/src/db/common/models/mod.rs +++ b/rust/sdk-processor/src/db/common/models/mod.rs @@ -1,3 +1,2 @@ pub mod backfill_processor_status; -pub mod events_models; pub mod processor_status; diff --git a/rust/sdk-processor/src/parquet_processors/mod.rs b/rust/sdk-processor/src/parquet_processors/mod.rs index d44c875b3..eec923a1d 100644 --- a/rust/sdk-processor/src/parquet_processors/mod.rs +++ b/rust/sdk-processor/src/parquet_processors/mod.rs @@ -27,7 +27,7 @@ use processor::{ parquet_transactions::Transaction as ParquetTransaction, parquet_write_set_changes::WriteSetChangeModel, }, - event_models::parquet_events::Event, + event_models::parquet_events::EventPQ, fungible_asset_models::{ parquet_v2_fungible_asset_activities::FungibleAssetActivity, parquet_v2_fungible_asset_balances::{ @@ -201,7 +201,7 @@ impl_parquet_trait!( ParquetTypeEnum::BlockMetadataTransactions ); impl_parquet_trait!(TableMetadata, ParquetTypeEnum::TableMetadata); -impl_parquet_trait!(Event, ParquetTypeEnum::Events); +impl_parquet_trait!(EventPQ, ParquetTypeEnum::Events); impl_parquet_trait!(UserTransaction, ParquetTypeEnum::UserTransactions); impl_parquet_trait!(AnsPrimaryNameV2, ParquetTypeEnum::AnsPrimaryNameV2); impl_parquet_trait!( @@ -274,7 +274,7 @@ pub enum ParquetTypeStructs { BlockMetadataTransaction(Vec), TableMetadata(Vec), // Events - Event(Vec), + Event(Vec), // User txn UserTransaction(Vec), // ANS types diff --git a/rust/sdk-processor/src/parquet_processors/parquet_events_processor.rs b/rust/sdk-processor/src/parquet_processors/parquet_events_processor.rs index f82546f4b..503dbd7b3 100644 --- a/rust/sdk-processor/src/parquet_processors/parquet_events_processor.rs +++ b/rust/sdk-processor/src/parquet_processors/parquet_events_processor.rs @@ -30,7 +30,7 @@ use aptos_indexer_processor_sdk::{ use parquet::schema::types::Type; use processor::{ bq_analytics::generic_parquet_processor::HasParquetSchema, - db::parquet::models::event_models::parquet_events::Event as EventPQ, + db::parquet::models::event_models::parquet_events::EventPQ, }; use std::{collections::HashMap, sync::Arc}; use tracing::{debug, info}; diff --git a/rust/sdk-processor/src/steps/events_processor/events_extractor.rs b/rust/sdk-processor/src/steps/events_processor/events_extractor.rs index 10db8b945..e09fbe6cf 100644 --- a/rust/sdk-processor/src/steps/events_processor/events_extractor.rs +++ b/rust/sdk-processor/src/steps/events_processor/events_extractor.rs @@ -1,15 +1,15 @@ -use crate::{ - db::common::models::events_models::events::EventModel, steps::MIN_TRANSACTIONS_PER_RAYON_JOB, -}; use aptos_indexer_processor_sdk::{ - aptos_protos::transaction::v1::{transaction::TxnData, Transaction}, + aptos_protos::transaction::v1::Transaction, traits::{async_step::AsyncRunType, AsyncStep, NamedStep, Processable}, types::transaction_context::TransactionContext, utils::errors::ProcessorError, }; use async_trait::async_trait; +use processor::db::{ + common::models::event_models::raw_events::parse_events, + postgres::models::events_models::events::EventPG, +}; use rayon::prelude::*; -use tracing::warn; pub struct EventsExtractor where @@ -18,55 +18,20 @@ where #[async_trait] impl Processable for EventsExtractor { type Input = Vec; - type Output = Vec; + type Output = Vec; type RunType = AsyncRunType; async fn process( &mut self, item: TransactionContext>, - ) -> Result>>, ProcessorError> { - // info!( - // start_version = item.start_version, - // end_version = item.end_version, - // step_name = self.name(), - // "Processing versions", - // ); - let events = item + ) -> Result>>, ProcessorError> { + let events: Vec = item .data .par_iter() - .with_min_len(MIN_TRANSACTIONS_PER_RAYON_JOB) - .map(|txn| { - let mut events = vec![]; - let txn_version = txn.version as i64; - let block_height = txn.block_height as i64; - let txn_data = match txn.txn_data.as_ref() { - Some(data) => data, - None => { - warn!( - transaction_version = txn_version, - "Transaction data doesn't exist" - ); - // PROCESSOR_UNKNOWN_TYPE_COUNT - // .with_label_values(&["EventsProcessor"]) - // .inc(); - return vec![]; - }, - }; - let default = vec![]; - let raw_events = match txn_data { - TxnData::BlockMetadata(tx_inner) => &tx_inner.events, - TxnData::Genesis(tx_inner) => &tx_inner.events, - TxnData::User(tx_inner) => &tx_inner.events, - TxnData::Validator(tx_inner) => &tx_inner.events, - _ => &default, - }; - - let txn_events = EventModel::from_events(raw_events, txn_version, block_height); - events.extend(txn_events); - events - }) + .map(|txn| parse_events(txn, "EventsProcessor")) .flatten() - .collect::>(); + .map(|e| e.into()) + .collect(); Ok(Some(TransactionContext { data: events, metadata: item.metadata, diff --git a/rust/sdk-processor/src/steps/events_processor/events_storer.rs b/rust/sdk-processor/src/steps/events_processor/events_storer.rs index f1a09e957..92c14f2a8 100644 --- a/rust/sdk-processor/src/steps/events_processor/events_storer.rs +++ b/rust/sdk-processor/src/steps/events_processor/events_storer.rs @@ -1,6 +1,5 @@ use crate::{ config::processor_config::DefaultProcessorConfig, - db::common::models::events_models::events::EventModel, utils::database::{execute_in_chunks, get_config_table_chunk_size, ArcDbPool}, }; use ahash::AHashMap; @@ -16,7 +15,7 @@ use diesel::{ query_builder::QueryFragment, ExpressionMethods, }; -use processor::schema; +use processor::{db::postgres::models::events_models::events::EventPG, schema}; use tracing::debug; pub struct EventsStorer @@ -37,7 +36,7 @@ impl EventsStorer { } fn insert_events_query( - items_to_insert: Vec, + items_to_insert: Vec, ) -> ( impl QueryFragment + diesel::query_builder::QueryId + Send, Option<&'static str>, @@ -58,27 +57,21 @@ fn insert_events_query( #[async_trait] impl Processable for EventsStorer { - type Input = Vec; + type Input = Vec; type Output = (); type RunType = AsyncRunType; async fn process( &mut self, - events: TransactionContext>, + events: TransactionContext>, ) -> Result>, ProcessorError> { - // tracing::info!( - // start_version = events.start_version, - // end_version = events.end_version, - // step_name = self.name(), - // "Processing versions", - // ); let per_table_chunk_sizes: AHashMap = self.processor_config.per_table_chunk_sizes.clone(); let execute_res = execute_in_chunks( self.conn_pool.clone(), insert_events_query, &events.data, - get_config_table_chunk_size::("events", &per_table_chunk_sizes), + get_config_table_chunk_size::("events", &per_table_chunk_sizes), ) .await; match execute_res { From 279ad33fb6c58868c4c1820c4ff86003a9d16fe9 Mon Sep 17 00:00:00 2001 From: bowenyang007 Date: Mon, 3 Feb 2025 07:26:25 -0800 Subject: [PATCH 2/4] fix asset_type_v1 null issue (#702) --- rust/processor/src/processors/fungible_asset_processor.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/processor/src/processors/fungible_asset_processor.rs b/rust/processor/src/processors/fungible_asset_processor.rs index dff9a364c..1ff563e27 100644 --- a/rust/processor/src/processors/fungible_asset_processor.rs +++ b/rust/processor/src/processors/fungible_asset_processor.rs @@ -50,8 +50,10 @@ use aptos_protos::transaction::v1::{transaction::TxnData, write_set_change::Chan use async_trait::async_trait; use chrono::NaiveDateTime; use diesel::{ + dsl::sql, pg::{upsert::excluded, Pg}, query_builder::QueryFragment, + sql_types::{Nullable, Text}, ExpressionMethods, }; use rayon::prelude::*; @@ -276,7 +278,8 @@ pub fn insert_current_unified_fungible_asset_balances_v2_query( .set( ( owner_address.eq(excluded(owner_address)), - asset_type_v1.eq(excluded(asset_type_v1)), + // This guarantees that asset_type_v1 will not be overridden to null + asset_type_v1.eq(sql::>("COALESCE(EXCLUDED.asset_type_v1, current_fungible_asset_balances.asset_type_v1)")), asset_type_v2.eq(excluded(asset_type_v2)), is_primary.eq(excluded(is_primary)), is_frozen.eq(excluded(is_frozen)), From 9bcb77e08004d4fdb8f26dc929c7a1355a9c5afd Mon Sep 17 00:00:00 2001 From: bowenyang007 Date: Mon, 3 Feb 2025 22:25:19 -0800 Subject: [PATCH 3/4] [FA migration] Dynamically process FA mapping (#698) --- .../fa_migration/coin_supply.json | 18 ++ .../current_fungible_asset_balances.json | 78 ++++++++ .../fungible_asset_activities.json | 74 ++++++++ .../fa_migration/fungible_asset_balances.json | 1 + .../fa_migration/fungible_asset_metadata.json | 36 ++++ .../src/models/events_models.rs | 3 +- .../src/models/fa_v2_models.rs | 11 +- .../fungible_asset_processor_tests.rs | 93 ++++++--- .../models/fungible_asset_models/mod.rs | 1 + .../raw_v2_fungible_asset_balances.rs | 110 ++--------- .../raw_v2_fungible_asset_to_coin_mappings.rs | 177 ++++++++++++++++++ .../down.sql | 4 + .../up.sql | 8 + .../models/fungible_asset_models/mod.rs | 1 + .../v2_fungible_asset_balances.rs | 89 --------- .../v2_fungible_asset_to_coin_mappings.rs | 34 ++++ rust/processor/src/db/postgres/schema.rs | 11 ++ .../processors/fungible_asset_processor.rs | 137 +++++++++++++- rust/processor/src/utils/util.rs | 10 +- .../parquet_fungible_asset_processor.rs | 7 +- .../processors/fungible_asset_processor.rs | 6 +- .../fungible_asset_extractor.rs | 56 +++++- .../fungible_asset_storer.rs | 38 +++- .../parquet_fa_extractor.rs | 35 +++- 24 files changed, 785 insertions(+), 253 deletions(-) create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/coin_supply.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/current_fungible_asset_balances.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_activities.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_balances.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_metadata.json create mode 100644 rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_to_coin_mappings.rs create mode 100644 rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/down.sql create mode 100644 rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/up.sql create mode 100644 rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_to_coin_mappings.rs diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/coin_supply.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/coin_supply.json new file mode 100644 index 000000000..9dbbad049 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/coin_supply.json @@ -0,0 +1,18 @@ +[ + { + "transaction_version": 1680592683, + "coin_type_hash": "91ceb1308a98389691e05158b07ed5f079ab78461a6bb8d5a4054b1bb5cb8bb6", + "coin_type": "0x1::aptos_coin::AptosCoin", + "supply": "111442820397688086", + "transaction_timestamp": "2024-09-05T11:52:16", + "transaction_epoch": 8371 + }, + { + "transaction_version": 1957950162, + "coin_type_hash": "91ceb1308a98389691e05158b07ed5f079ab78461a6bb8d5a4054b1bb5cb8bb6", + "coin_type": "0x1::aptos_coin::AptosCoin", + "supply": "112743621173497128", + "transaction_timestamp": "2024-11-23T22:32:30", + "transaction_epoch": 9331 + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/current_fungible_asset_balances.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/current_fungible_asset_balances.json new file mode 100644 index 000000000..dae8cce2c --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/current_fungible_asset_balances.json @@ -0,0 +1,78 @@ +[ + { + "storage_id": "0x1734f2619bb775e59d04469663b9c05614b5ada6afbf2f750095f43cb8d41e40", + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "115245132", + "amount_v2": null, + "amount": "115245132", + "last_transaction_version_v1": 1957950162, + "last_transaction_version_v2": null, + "last_transaction_version": 1957950162, + "last_transaction_timestamp_v1": "2024-11-23T22:32:30", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2024-11-23T22:32:30", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0xa1ddab83c8209aabffc66264730ba95bcc7d7a7966192c00d800ab734729622c", + "owner_address": "0x7ae7cc51c4fab67181a969f53a5d01a292dad0baf259c92c4e1a13f056768e1c", + "asset_type_v2": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "asset_type_v1": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL", + "is_primary": true, + "is_frozen": false, + "amount_v1": null, + "amount_v2": "431101567356", + "amount": "431101567356", + "last_transaction_version_v1": null, + "last_transaction_version_v2": 1957950162, + "last_transaction_version": 1957950162, + "last_transaction_timestamp_v1": null, + "last_transaction_timestamp_v2": "2024-11-23T22:32:30", + "last_transaction_timestamp": "2024-11-23T22:32:30", + "token_standard": "v1", + "asset_type": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL" + }, + { + "storage_id": "0xd29b17433643769f27d89851553ec2ab5e4204b66eceecb6eec774668117c3e0", + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "asset_type_v2": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "asset_type_v1": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL", + "is_primary": true, + "is_frozen": false, + "amount_v1": null, + "amount_v2": "3643926", + "amount": "3643926", + "last_transaction_version_v1": null, + "last_transaction_version_v2": 1957950162, + "last_transaction_version": 1957950162, + "last_transaction_timestamp_v1": null, + "last_transaction_timestamp_v2": "2024-11-23T22:32:30", + "last_transaction_timestamp": "2024-11-23T22:32:30", + "token_standard": "v1", + "asset_type": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL" + }, + { + "storage_id": "0xfa4e15c47bc6b38cc404cf08e30e69fd14ff2cc82b136ef3630445a66605653c", + "owner_address": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "997323833", + "amount_v2": null, + "amount": "997323833", + "last_transaction_version_v1": 1680592683, + "last_transaction_version_v2": null, + "last_transaction_version": 1680592683, + "last_transaction_timestamp_v1": "2024-09-05T11:52:16", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2024-09-05T11:52:16", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_activities.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_activities.json new file mode 100644 index 000000000..ecd37babc --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_activities.json @@ -0,0 +1,74 @@ +[ + { + "transaction_version": 1680592683, + "event_index": -1, + "owner_address": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06", + "storage_id": "0xfa4e15c47bc6b38cc404cf08e30e69fd14ff2cc82b136ef3630445a66605653c", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "205700", + "type_": "0x1::aptos_coin::GasFeeEvent", + "is_gas_fee": true, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_mkl_token::initialize_module", + "block_height": 222731447, + "token_standard": "v1", + "transaction_timestamp": "2024-09-05T11:52:16", + "storage_refund_amount": "47280" + }, + { + "transaction_version": 1957950162, + "event_index": -1, + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "storage_id": "0x1734f2619bb775e59d04469663b9c05614b5ada6afbf2f750095f43cb8d41e40", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "2100", + "type_": "0x1::aptos_coin::GasFeeEvent", + "is_gas_fee": true, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_liquidity_auction::claim_mkl_reward", + "block_height": 256094969, + "token_standard": "v1", + "transaction_timestamp": "2024-11-23T22:32:30", + "storage_refund_amount": "0" + }, + { + "transaction_version": 1957950162, + "event_index": 0, + "owner_address": "0x7ae7cc51c4fab67181a969f53a5d01a292dad0baf259c92c4e1a13f056768e1c", + "storage_id": "0xa1ddab83c8209aabffc66264730ba95bcc7d7a7966192c00d800ab734729622c", + "asset_type": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "is_frozen": null, + "amount": "3643926", + "type_": "0x1::fungible_asset::Withdraw", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_liquidity_auction::claim_mkl_reward", + "block_height": 256094969, + "token_standard": "v2", + "transaction_timestamp": "2024-11-23T22:32:30", + "storage_refund_amount": "0" + }, + { + "transaction_version": 1957950162, + "event_index": 1, + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "storage_id": "0xd29b17433643769f27d89851553ec2ab5e4204b66eceecb6eec774668117c3e0", + "asset_type": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "is_frozen": null, + "amount": "3643926", + "type_": "0x1::fungible_asset::Deposit", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_liquidity_auction::claim_mkl_reward", + "block_height": 256094969, + "token_standard": "v2", + "transaction_timestamp": "2024-11-23T22:32:30", + "storage_refund_amount": "0" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_balances.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_balances.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_balances.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_metadata.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_metadata.json new file mode 100644 index 000000000..9f8c23199 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration/fungible_asset_metadata.json @@ -0,0 +1,36 @@ +[ + { + "asset_type": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL", + "creator_address": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06", + "name": "MKL", + "symbol": "MKL", + "decimals": 6, + "icon_uri": null, + "project_uri": null, + "last_transaction_version": 1680592683, + "last_transaction_timestamp": "2024-09-05T11:52:16", + "supply_aggregator_table_handle_v1": null, + "supply_aggregator_table_key_v1": null, + "token_standard": "v1", + "maximum_v2": null, + "is_token_v2": null, + "supply_v2": null + }, + { + "asset_type": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "creator_address": "0x000000000000000000000000000000000000000000000000000000000000000a", + "name": "MKL", + "symbol": "MKL", + "decimals": 6, + "icon_uri": "", + "project_uri": "", + "last_transaction_version": 1957950162, + "last_transaction_timestamp": "2024-11-23T22:32:30", + "supply_aggregator_table_handle_v1": null, + "supply_aggregator_table_key_v1": null, + "token_standard": "v2", + "maximum_v2": "340282366920938463463374607431768211455", + "is_token_v2": null, + "supply_v2": "96895109643615" + } +] \ No newline at end of file diff --git a/rust/integration-tests/src/models/events_models.rs b/rust/integration-tests/src/models/events_models.rs index 3255dd3ee..c5cd8b0e8 100644 --- a/rust/integration-tests/src/models/events_models.rs +++ b/rust/integration-tests/src/models/events_models.rs @@ -9,8 +9,7 @@ use processor::schema::events; use serde::{Deserialize, Serialize}; /** * Event model -* this is created // b/c there is inserated_at field which isn't defined in the Event struct, we can't just load the events directly without specifying the fields. -* TODO: make this more generic to load all fields, then we should be able to run tests for all processor in one test case. +* this is created b/c there is inserated_at field which isn't defined in the Event struct, we can't just load the events directly without specifying the fields. */ #[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize, Queryable)] #[diesel(primary_key(transaction_version, event_index))] diff --git a/rust/integration-tests/src/models/fa_v2_models.rs b/rust/integration-tests/src/models/fa_v2_models.rs index 3cfaa9a21..c3fdf4dc2 100644 --- a/rust/integration-tests/src/models/fa_v2_models.rs +++ b/rust/integration-tests/src/models/fa_v2_models.rs @@ -8,7 +8,7 @@ use diesel::{Identifiable, Insertable, Queryable}; use field_count::FieldCount; use processor::schema::{ coin_supply, current_fungible_asset_balances, fungible_asset_activities, - fungible_asset_balances, fungible_asset_metadata, + fungible_asset_balances, fungible_asset_metadata, fungible_asset_to_coin_mappings, }; use serde::{Deserialize, Serialize}; @@ -110,3 +110,12 @@ pub struct CoinSupply { pub transaction_epoch: i64, pub inserted_at: chrono::NaiveDateTime, } + +#[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)] +#[diesel(primary_key(coin_type))] +#[diesel(table_name = fungible_asset_to_coin_mappings)] +pub struct FungibleAssetToCoinMapping { + pub coin_type: String, + pub fungible_asset_metadata_address: String, + pub last_transaction_version: i64, +} diff --git a/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs b/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs index 047270658..9ee3e1ee0 100644 --- a/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs @@ -50,7 +50,9 @@ mod sdk_fungible_asset_processor_tests { }; use aptos_indexer_test_transactions::json_transactions::generated_transactions::{ IMPORTED_DEVNET_TXNS_78753811_COIN_TRANSFER_WITH_V2_EVENTS, + IMPORTED_MAINNET_TXNS_1680592683_FA_MIGRATION_COIN_INFO, IMPORTED_MAINNET_TXNS_1737056775_COIN_TRANSFER_BURN_EVENT, + IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY, IMPORTED_MAINNET_TXNS_2186504987_COIN_STORE_DELETION_NO_EVENT, IMPORTED_MAINNET_TXNS_255894550_STORAGE_REFUND, IMPORTED_MAINNET_TXNS_508365567_FA_V1_EVENTS, @@ -69,8 +71,8 @@ mod sdk_fungible_asset_processor_tests { // Test case for processing a specific testnet transaction (Validator Transaction) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_validator_txn() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_5523474016_VALIDATOR_TXN, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_5523474016_VALIDATOR_TXN], Some("validator_txn_test".to_string()), ) .await; @@ -79,8 +81,8 @@ mod sdk_fungible_asset_processor_tests { // Test case for processing another testnet transaction (Coin Register) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_register_txn() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_5979639459_COIN_REGISTER, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_5979639459_COIN_REGISTER], Some("coin_register_txn_test".to_string()), ) .await; @@ -88,8 +90,8 @@ mod sdk_fungible_asset_processor_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_fa_activities_txn() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_5992795934_FA_ACTIVITIES, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_5992795934_FA_ACTIVITIES], Some("fa_activities_txn_test".to_string()), ) .await; @@ -110,8 +112,8 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_and_fa_transfers() { - process_single_testnet_fa_txns( - IMPORTED_MAINNET_TXNS_999929475_COIN_AND_FA_TRANSFERS, + process_fa_txns( + &[IMPORTED_MAINNET_TXNS_999929475_COIN_AND_FA_TRANSFERS], Some("coin_and_fa_transfers_test".to_string()), ) .await; @@ -126,8 +128,8 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_v1_events() { - process_single_testnet_fa_txns( - IMPORTED_MAINNET_TXNS_508365567_FA_V1_EVENTS, + process_fa_txns( + &[IMPORTED_MAINNET_TXNS_508365567_FA_V1_EVENTS], Some("v1_events_test".to_string()), ) .await; @@ -140,8 +142,8 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_v2_frozen_event() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_1200394037_FA_V2_FROZEN_EVENT, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_1200394037_FA_V2_FROZEN_EVENT], Some("v2_frozen_event_test".to_string()), ) .await; @@ -155,8 +157,8 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_concurrent_fa() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_2646510387_CONCURRENT_FA, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_2646510387_CONCURRENT_FA], Some("concurrent_fa_test".to_string()), ) .await; @@ -166,8 +168,8 @@ mod sdk_fungible_asset_processor_tests { /// Validates the handling of updated coin event formats #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_v2_events() { - process_single_testnet_fa_txns( - IMPORTED_DEVNET_TXNS_78753811_COIN_TRANSFER_WITH_V2_EVENTS, + process_fa_txns( + &[IMPORTED_DEVNET_TXNS_78753811_COIN_TRANSFER_WITH_V2_EVENTS], Some("coin_v2_events".to_string()), ) .await; @@ -175,8 +177,8 @@ mod sdk_fungible_asset_processor_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_store_deletion_no_event() { - process_single_testnet_fa_txns( - IMPORTED_MAINNET_TXNS_2186504987_COIN_STORE_DELETION_NO_EVENT, + process_fa_txns( + &[IMPORTED_MAINNET_TXNS_2186504987_COIN_STORE_DELETION_NO_EVENT], Some("coin_store_deletion_no_event".to_string()), ) .await; @@ -186,8 +188,8 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of burning tokens from secondary stores #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_secondary_store_burnt() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_4462417704_SECONDARY_STORE_BURNT, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_4462417704_SECONDARY_STORE_BURNT], Some("secondary_store_burnt".to_string()), ) .await; @@ -197,8 +199,8 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of isolated gas events #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_gas_event_when_events_is_empty() { - process_single_testnet_fa_txns( - IMPORTED_TESTNET_TXNS_646928741_NO_EVENTS, + process_fa_txns( + &[IMPORTED_TESTNET_TXNS_646928741_NO_EVENTS], Some("gas_event_when_events_is_empty".to_string()), ) .await; @@ -208,8 +210,8 @@ mod sdk_fungible_asset_processor_tests { /// Validates handling of burn operations during coin transfers #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_transfer_burn_event() { - process_single_testnet_fa_txns( - IMPORTED_MAINNET_TXNS_1737056775_COIN_TRANSFER_BURN_EVENT, + process_fa_txns( + &[IMPORTED_MAINNET_TXNS_1737056775_COIN_TRANSFER_BURN_EVENT], Some("coin_transfer_burn_event".to_string()), ) .await; @@ -219,8 +221,8 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of batch transfers #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_multiple_transfer_event() { - process_single_testnet_fa_txns( - IMPORTED_MAINNET_TXNS_550582915_MULTIPLE_TRANSFER_EVENT, + process_fa_txns( + &[IMPORTED_MAINNET_TXNS_550582915_MULTIPLE_TRANSFER_EVENT], Some("multiple_transfer_event".to_string()), ) .await; @@ -230,19 +232,50 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of storage refund mechanics #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_storage_refund() { - process_single_testnet_fa_txns( - IMPORTED_MAINNET_TXNS_255894550_STORAGE_REFUND, + process_fa_txns( + &[IMPORTED_MAINNET_TXNS_255894550_STORAGE_REFUND], Some("storage_refund".to_string()), ) .await; } + /// Test FA migration. It's a 2 transaction test where the + /// first transaction creates a CoinInfo and the second + /// processes an FA version of that coin + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_fungible_asset_processor_fa_migration_same_batch() { + process_fa_txns( + &[ + IMPORTED_MAINNET_TXNS_1680592683_FA_MIGRATION_COIN_INFO, + IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY, + ], + Some("fa_migration".to_string()), + ) + .await; + } + + // TODO: I really want to make this work but it doesn't right now + // #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + // async fn test_fungible_asset_processor_fa_migration_different_batch() { + // process_fa_txns( + // &[IMPORTED_MAINNET_TXNS_1680592683_FA_MIGRATION_COIN_INFO], + // Some("fa_migration_2_batch".to_string()), + // ) + // .await; + + // process_fa_txns( + // &[IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY], + // Some("fa_migration_2_batch".to_string()), + // ) + // .await; + // } + // Helper function to abstract out the transaction processing - async fn process_single_testnet_fa_txns(txn: &[u8], test_case_name: Option) { + async fn process_fa_txns(txns: &[&[u8]], test_case_name: Option) { let (diff_flag, custom_output_path) = get_test_config(); let output_path = custom_output_path.unwrap_or_else(|| DEFAULT_OUTPUT_FOLDER.to_string()); - let (db, mut test_context) = setup_test_environment(&[txn]).await; + let (db, mut test_context) = setup_test_environment(txns).await; let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = diff --git a/rust/processor/src/db/common/models/fungible_asset_models/mod.rs b/rust/processor/src/db/common/models/fungible_asset_models/mod.rs index 622093525..02acd88d5 100644 --- a/rust/processor/src/db/common/models/fungible_asset_models/mod.rs +++ b/rust/processor/src/db/common/models/fungible_asset_models/mod.rs @@ -3,4 +3,5 @@ pub mod raw_v2_fungible_asset_activities; pub mod raw_v2_fungible_asset_balances; +pub mod raw_v2_fungible_asset_to_coin_mappings; pub mod raw_v2_fungible_metadata; diff --git a/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_balances.rs b/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_balances.rs index 8ce21e57a..b67964b50 100644 --- a/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_balances.rs +++ b/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_balances.rs @@ -5,7 +5,12 @@ #![allow(clippy::extra_unused_lifetimes)] #![allow(clippy::unused_unit)] -use super::raw_v2_fungible_asset_activities::AddressToCoinType; +use super::{ + raw_v2_fungible_asset_activities::AddressToCoinType, + raw_v2_fungible_asset_to_coin_mappings::{ + FungibleAssetToCoinMappings, RawFungibleAssetToCoinMapping, + }, +}; use crate::{ db::{ common::models::{ @@ -27,7 +32,6 @@ use crate::{ use ahash::AHashMap; use aptos_protos::transaction::v1::{DeleteResource, WriteResource}; use bigdecimal::{BigDecimal, Zero}; -use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -35,93 +39,6 @@ use std::str::FromStr; pub type CurrentUnifiedFungibleAssetMapping = AHashMap; -lazy_static!( - pub static ref METADATA_TO_COIN_TYPE_MAPPING: AHashMap<&'static str, &'static str> = vec![ - ("0x000000000000000000000000000000000000000000000000000000000000000a", "0x1::aptos_coin::AptosCoin"), - ("0x377adc4848552eb2ea17259be928001923efe12271fef1667e2b784f04a7cf3a", "0x7fd500c11216f0fe3095d0c4b8aa4d64a4e2e04f83758462f2b127255643615::thl_coin::THL"), - ("0x0009da434d9b873b5159e8eeed70202ad22dc075867a7793234fbc981b63e119", "0xe4ccb6d39136469f376242c31b34d10515c8eaaa38092f804db8e08a8f53c5b2::assets_v1::EchoCoin002"), - ("0xe568e9322107a5c9ba4cbd05a630a5586aa73e744ada246c3efb0f4ce3e295f3", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDT"), - ("0xd4c0be6af89a42d78fb728dd57096eda717d7044c2dd635a01417662c33614fc", "0x55987edfab9a57f69bac759674f139ae473b5e09a9283848c1f87faf6fc1e789::shrimp::ShrimpCoin"), - ("0xa259be733b6a759909f92815927fa213904df6540519568692caf0b068fe8e62", "0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::amapt_token::AmnisApt"), - ("0xb614bfdf9edc39b330bbf9c3c5bcd0473eee2f6d4e21748629cc367869ece627", "0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::stapt_token::StakedApt"), - ("0x2b3be0a97a73c87ff62cbdd36837a9fb5bbd1d7f06a73b7ed62ec15c5326c1b8", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDC"), - ("0xb27b0c6b60772f0fc804ec1cd3339f552badf9bd1e125a7dd700d8eb11248ef1", "0x73eb84966be67e4697fc5ae75173ca6c35089e802650f75422ab49a8729704ec::coin::DooDoo"), - ("0x5915ae0eae3701833fa02e28bf530bc01ca96a5f010ac8deecb14c7a92661368", "0x4fbed3f8a3fd8a11081c8b6392152a8b0cb14d70d0414586f0c9b858fcd2d6a7::UPTOS::UPTOS"), - ("0x6dba1728c73363be1bdd4d504844c40fbb893e368ccbeff1d1bd83497dbc756d", "0xe50684a338db732d8fb8a3ac71c4b8633878bd0193bca5de2ebc852a83b35099::propbase_coin::PROPS"), - ("0xad18575b0e51dd056e1e082223c0e014cbfe4b13bc55e92f450585884f4cf951", "0x159df6b7689437016108a019fd5bef736bac692b6d4a1f10c941f6fbb9a74ca6::oft::CakeOFT"), - ("0xae02f68520afd221a5cd6fda6f5500afedab8d0a2e19a916d6d8bc2b36e758db", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WETH"), - ("0x0a9ce1bddf93b074697ec5e483bc5050bc64cff2acd31e1ccfd8ac8cae5e4abe", "0xfaf4e633ae9eb31366c9ca24214231760926576c7b625313b3688b5e900731f6::staking::StakedThalaAPT"), - ("0xd1bec63fa44dc3f3f5742c3f3a4afc3baed00505efbe955dfe6e5f9d306c67a5", "0xd0b4efb4be7c3508d9a26a9b5405cf9f860d0b9e5fe2f498b90e68b8d2cedd3e::aptos_launch_token::AptosLaunchToken"), - ("0xb81588af2f7d291e8e57f673ec74d4a38f0654633ad6bbeb1cfd4bb0550ae0df", "0x4def3d3dee27308886f0a3611dd161ce34f977a9a5de4e80b237225923492a2a::coin::T"), - ("0x02370cc1d995f3aadd337c1c6c63834ad8d2bd0cdc70bc8dff81de463e18b159", "0x53a30a6e5936c0a4c5140daed34de39d17ca7fcae08f947c02e979cef98a3719::coin::LSD"), - ("0xbe34691f0388bbca83bafab87399aeb756284e11a2872f1ae74218451cb3899f", "0xdd89c0e695df0692205912fb69fc290418bed0dbe6e4573d744a6d5e6bab6c13::coin::T"), - ("0x96d1ccca420ebc20fc8af6cacb864e44856ca879c6436d4e9be2b0a4b99bf852", "0x27fafcc4e39daac97556af8a803dbb52bcb03f0821898dc845ac54225b9793eb::move_coin::MoveCoin"), - ("0x94ed76d3d66cb0b6e7a3ab81acf830e3a50b8ae3cfb9edc0abea635a11185ff4", "0x6f986d146e4a90b828d8c12c14b6f4e003fdff11a8eecceceb63744363eaac01::mod_coin::MOD"), - ("0xa743351ed4889845737082ab9fcd42c21270647e2c6f342c509320e974035ed2", "0xd11107bdf0d6d7040c6c0bfbdecb6545191fdf13e8d8d259952f53e1713f61b5::staked_coin::StakedAptos"), - ("0x41944cf1d4dac152d692644944e2cc49ee81fafdfb37abd541d06388ec3f7eda", "0xe88ae9670071da40a9a6b1d97aab8f6f1898fdc3b8f1c1038b492dfad738448b::coin::Donk"), - ("0x4c3efb98d8d3662352f331b3465c6df263d1a7e84f002844348519614a5fea30", "0x63be1898a424616367e19bbd881f456a78470e123e2770b5b5dcdceb61279c54::movegpt_token::MovegptCoin"), - ("0x5eea0061714d46da0ccfd088df4fb1a2ea26c2421e592ade9dacc21cdb60e056", "0x1000000fa32d122c18a6a31c009ce5e71674f22d06a581bb0a15575e6addadcc::usda::USDA"), - ("0x54fc0d5fa5ad975ede1bf8b1c892ae018745a1afd4a4da9b70bb6e5448509fc0", "0x5e156f1207d0ebfa19a9eeff00d62a282278fb8719f4fab3a586a0a2c0fffbea::coin::T"), - ("0xc448a48da1ed6f6f378bb82ece996be8b5fc8dd1ea851ea2c023de17714dd747", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::UsdtCoin"), - ("0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL"), - ("0x1912eb1a5f8f0d4c72c1687eb199b7f9d2df34da5931ec96830c557f7abaa0ad", "0x84d7aeef42d38a5ffc3ccef853e1b82e4958659d16a7de736a29c55fbbeb0114::staked_aptos_coin::StakedAptosCoin"), - ("0xb5f9a9ff6f815150af83b96b15e4f85e4e3b9e92e3fd17a414cd755c4aa49513", "0x7c0322595a73b3fc53bb166f5783470afeb1ed9f46d1176db62139991505dc61::abel_coin::AbelCoin"), - ("0x6704464238d73a679486420aab91a8a2a01feb9d700e8ba3332aa3e41d3eab62", "0xa2eda21a58856fda86451436513b867c97eecb4ba099da5775520e0f7492e852::coin::T"), - ("0xd47b65c45f5260c4f3c5de3f32ddaeabf7eab56c9493a7a955dff7f70ba8afaf", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::BusdCoin"), - ("0x92410a41654236295001f06375afbb1786dbd14bc1c42a33bfcf50204c248bb7", "0xcc8a89c8dce9693d354449f1f73e60e14e347417854f029db5bc8e7454008abb::coin::T"), - ("0x59c3a92ab1565a14d4133eb2a3418604341d37da47698a0e853c7bb22c342c55", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::DaiCoin"), - ("0x1fe81b3886ff129d42064f7ee934013de7ef968cb8f47adb5f7210192bcd4a23", "0xc26a8eda1c3ab69a157815183ddda88c89d6758ee491dd1647a70af2907ce074::coin::Chewy"), - ("0xa64d2d6f5e26daf6a3552f51d4110343b1a8c8046d0a9e72fa4086a337f3236c", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WBTC"), - ("0x2329a8351b28aa3672329217a949a9ab39d7f24534324c2eeb8b18f69e7a6fb1", "0x4ed27736e724e403f9b4645ffef0ae86fd149503f45b37c428ffabd7e46e5b05::core::RenegadeCoin"), - ("0xa79e44c5cdf8a0eb835a265a20adab56c8cdf169a2a25daa3b1d71c557b9ec59", "0xd6d6372c8bde72a7ab825c00b9edd35e643fb94a61c55d9d94a9db3010098548::USDC::Coin"), - ("0xde368b120e750bbb0d8799356ea23511306ff19edd28eed15fe7b6cc72b04388", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDD"), - ("0x41dfe1fb3d33d4d9d0b460f03ce1c0a6af6520dd8bdc0f204583c4987faf81de", "0x268d4a7a2ad93274edf6116f9f20ad8455223a7ab5fc73154f687e7dbc3e3ec6::LOON::LOON"), - ("0x4a884be56ef4c11a27162bf30b32e8e3615dcb3df0fc1777c8eb69c1991f34d0", "0xae478ff7d83ed072dbc5e264250e67ef58f57c99d89b447efd8a0a2e8b2be76e::coin::T"), - ("0xdc5e054538ba5e183d5aa197b01f327cf84aace749dc8fa2fe87bb5ec9bfe35a", "0xe1bfc010d2bdd576036f4c1f3ea7d547f19188f5b78075dd961420d843ee914c::brew_coin::BrewCoin"), - ("0xd2f2fd4a3df494042cf24c3b8c1316be8bab7ebac228be77cc0f19fcd885c666", "0x1fc2f33ab6b624e3e632ba861b755fd8e61d2c2e6cf8292e415880b4c198224d::apt20::APTS"), - ("0x9f0de082b2d4506de8b546308d4fd58bdd5ef5981097abeff40635e979874c9e", "0x65957cb717d1ec5e13c003cbad0d20d8f7f95236ea0f8bb8c419e533eda73890::TOAST::TOAST"), - ("0xc8e09f0daa8f0143318c965b43cecad55eb1a4f26ea57677fcf44c36975fe28c", "0x665d06fcd9c94430099f82973f2a5e5f13142e42fa172e72ce14f51a64bd8ad9::coin_mbx::MBX"), - ("0x5b5d60e20f3684ce19d3fd3a99ed2b2a8722b043fd67cea80ea4bc0a4749e883", "0x7c2aaaaf3f019bbf7f02beed21fc4ec352cc38272f93cb11e61ec7c89bfe5f4b::xbtc::XBTC"), - ("0x69ef9f94420a1d287892fb42450ca5777984c1c22cc886407726482480276ec1", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::WbtcCoin"), - ("0x78d37cff9f42109ce68cd73edb9ef24bb03aad697d7b6449a544701e425befbf", "0x967adbf2e05fe665ab86a3bf2c4acfa39fbf62097963474ef70a0786dae8cfa2::NRUH::NRUH"), - ("0xa36e2774e4db37934a3e27e2df7b39be5e6fcb4b7840319336fb98ffdf3d613a", "0xc82974034820c34f065f948f1bee473d481bf99fde2d23e981043e5038cb36be::WOOF::Woof"), - ("0x37bdfd533a28c38ba6f2963e3f2ab881b3826d952ea3d4ca03020e0d2735348d", "0xccc9620d38c4f3991fa68a03ad98ef3735f18d04717cb75d7a1300dd8a7eed75::coin::T"), - ("0xeb73df9d3ba3fbc2538d2e7f4a2dac9718b48b07f65596a9c7cc84d978e3d6cd", "0x2dcbc03740a6fa2efee926b9df329184cce357d0573bdab09930f4d48e61a4c8::STOS::STOS"), - ("0xcd70630fb90cab716ab01a7884821f86dceb1bbb09a89683b5c22c5462503f51", "0xdcfa079344261bfde45e7f6281df091743b8d3098bf9e26e1c0212fc5b070621::zaaptos_token::ZaaptosCoin"), - ("0x1b976ce1a6bf4a37057166f52646c879f37cb0712eb2fd4005e54b9c929b7171", "0x380b3422c7d2c28f29776cf1a234b98bc514b6d8c1cf16e1b9123cb1acb6203::plant_coin::PlantCoin"), - ("0xfd1f22b455e2d095a1ec497ecc0a11db86173d0271557ca6c16baa9d66466a75", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x1::aptos_coin::AptosCoin, 0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDC>"), - ("0x0943bf6e5329167f459a4ae7efa93336f49ef08a5aff65f8c70133f0a0725ef2", "0x91b54cb4441c88fa21b7ca5b8b860e8b6fe726c23866bed91999823e65c1026d::GEMKRW::GEMKRW"), - ("0xf92047adba5ec4a21ad076b19a0c8806b195435696d30dc3f43781d1e6d91563", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::UsdcCoin"), - ("0xf0876baf6f8c37723f0e9d9c1bbad1ccb49324c228bcc906e2f1f5a9e139eda1", "0xcfea864b32833f157f042618bd845145256b1bf4c0da34a7013b76e42daa53cc::usdy::USDY"), - ("0xf7833d21f83a19548c81e8fd17d8bde4a6e8cc3fbb1ffb97973e06e261c75dee", "0x5c738a5dfa343bee927c39ebe85b0ceb95fdb5ee5b323c95559614f5a77c47cf::Aptoge::Aptoge"), - ("0xa4607412abfc37ec0b6fd6e102f5f0b7989f59fd44ff5d374cbe360ffbecdfff", "0xf891d2e004973430cc2bbbee69f3d0f4adb9c7ae03137b4579f7bb9979283ee6::APTOS_FOMO::APTOS_FOMO"), - ("0x109492c6323a413d605f5768127b11ef28d5805a818b355b0c9ebcb1995fcf81", "0x881ac202b1f1e6ad4efcff7a1d0579411533f2502417a19211cfc49751ddb5f4::coin::MOJO"), - ("0xfbd6406c12cab2aef728c917a365cdb73883213f74af5e8a46c8fbd77b623ee7", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::WethCoin"), - ("0xc40443d625f94ddec95a76bcf2534eda394bf67713b93f08eb202026e2aaa66a", "0xdf3d5eb83df80dfde8ceb1edaa24d8dbc46da6a89ae134a858338e1b86a29e38::coin::Returd"), - ("0x08bbc1e07f934be0862be6df1477dbab54d6ccdf594f1569a64fa2941cbfe368", "0x198e4a77b72cbcac7465e774e99d2ba552053ca57b0759ea3c008433742b4e4f::PEPE::Pepe"), - ("0x290c792f89a47cd7280e0b9035fa8b2876ab4298f0135d4a2c88e77257681ea1", "0x2778b277644d375721766abfff0df2adca795f6cbae9f02ff1c95ce9adb6ee28::aptos_shiba_coin::AptosShibaCoin"), - ("0xc73b3f454576b00d4d05393ff427537eda42f791350f30ce1f566448b5798644", "0x66302f3c648890f70ca3fafc42c919483945f3ba155101bc2e149e38a8b93afc::toss_coin::TossCoin"), - ("0x3fb0cd30095fc85c77eb5cb9edcdbead3cef34e449b1a6f07729282120bc6383", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::BnbCoin"), - ("0xa0d9d647c5737a5aed08d2cfeb39c31cf901d44bc4aa024eaa7e5e68b804e011", "0xfaf4e633ae9eb31366c9ca24214231760926576c7b625313b3688b5e900731f6::staking::ThalaAPT"), - ("0x0dcee4819a7b45113c6e44a157a11866f3366a7c93f79ba5acdf27f6fb8ce301", "0xbe3c4b493632b4d776d56e19d91dcfb34f591f759f6b57f8632d455360da503c::dumdum_coin::DumdumCoin"), - ("0x23813a24e98215ab541051432b734baecaa3737019a4891c37034f88d9944960", "0x407a220699982ebb514568d007938d2447d33667e4418372ffec1ddb24491b6c::coin::T"), - ("0xb56df862320ff2dc317e147c870ad09f12711a5e02c6245438f827e6c54188b4", "0x5c738a5dfa343bee927c39ebe85b0ceb95fdb5ee5b323c95559614f5a77c47cf::AptSwap::AptSwapGovernance"), - ("0xf5d23515c4454652c38219aec5f1a0720207dc1f5d2e5140b94608f9ce821a36", "0x84edd115c901709ef28f3cb66a82264ba91bfd24789500b6fd34ab9e8888e272::coin::DLC"), - ("0xcc71496adf8086dcc62bbe4d975718f09de8cc42629aacd8df84df514cbac154", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x1::aptos_coin::AptosCoin, 0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDT>"), - ("0xfad230e7d9df2baf83a68b6f50217ed3c06da593e766970a885965b43b894a04", "0xada35ada7e43e2ee1c39633ffccec38b76ce702b4efc2e60b50f63fbe4f710d8::apetos_token::ApetosCoin"), - ("0x8d55e255fcf4142a8008b1b5ca0ec9efb964f8a914807c0bf279f6c3af3ef955", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::amapt_token::AmnisApt, 0x1::aptos_coin::AptosCoin>"), - ("0x66037303c053e2bd0d5af444a9a4792e5a2b56633849e57259e575349d871d04", "0x1eb35b3b9e911ca2093970ae5abfb970dbec54597d43954bb53b09d2e8356cc3::BMTAmm::BMTAmm"), - ("0xcab64ed0d956462e9b8ba7c340fdb8b9ab52da1503f37b522288bc0c5bf944de", "0x389dbbc6884a1d5b1ab4e1df2913a8c1b01251e50aed377554372b2842c5e3ef::EONcoin::EONCoin"), - ("0x5486d29c4fceec48c55e88a700eddfdf5be8663a2a873ac0d2baac21cd78b390", "0xacd014e8bdf395fa8497b6d585b164547a9d45269377bdf67c96c541b7fec9ed::coin::T"), - ("0x9660042a7c01d776938184278381d24c7009ca385d9a59cf9b22691f97615960", "0x16fe2df00ea7dde4a63409201f7f4e536bde7bb7335526a35d05111e68aa322c::AnimeCoin::ANI"), - ("0x4ddd6e6dfff083e2e4981cf959384e6aec18a9c62cc4694e8aab950c07296208", "0xcc78307c77f1c2c0fdfee17269bfca7876a0b35438c3442417480c0d5c370fbc::AptopadCoin::APD"), - ("0x8ddd74585cfe3249d71532a8c96fbfc3a1d1134357ec1eaf0411db953c6b942b", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WETH, 0x1::aptos_coin::AptosCoin>"), - ("0x70003a36f53ed8142a7a530415982d466bc16cdcf2be2599c08211105bd9364d", "0x26f03cd414cdcae387961058ab5523a4e64559f4b0853c46c9b3c4e01fd8af9b::usdy_staging::USDY_STAGING"), - ("0x64196428c8f492b189a1f69234e12b161adf729fc42679cb30219a59f8114f8c", "0x26f03cd414cdcae387961058ab5523a4e64559f4b0853c46c9b3c4e01fd8af9b::usdy::USDY"), - ("0x56998a39007a9b431fa9d98bcc57de14f2f357846723a1ad94dfb4c3c965a3a3", "0x14b0ef0ec69f346bea3dfa0c5a8c3942fb05c08760059948f9f24c02cd0d4fd8::mover_token::Mover"), -].iter().cloned().collect(); - ); - #[derive(Clone, Debug, Deserialize, Serialize)] pub struct RawFungibleAssetBalance { pub transaction_version: i64, @@ -187,6 +104,7 @@ pub fn get_primary_fungible_store_address( impl RawCurrentUnifiedFungibleAssetBalance { pub fn from_fungible_asset_balances( fungible_asset_balances: &[RawFungibleAssetBalance], + fa_to_coin_mapping: Option<&FungibleAssetToCoinMappings>, ) -> ( CurrentUnifiedFungibleAssetMapping, CurrentUnifiedFungibleAssetMapping, @@ -196,7 +114,7 @@ impl RawCurrentUnifiedFungibleAssetBalance { let mut v2_balances: CurrentUnifiedFungibleAssetMapping = AHashMap::new(); for balance in fungible_asset_balances.iter() { - let unified_balance = Self::from(balance); + let unified_balance = Self::from_balance(balance, fa_to_coin_mapping); match TokenStandard::from_str(&balance.token_standard).expect("Invalid token standard") { TokenStandard::V1 => { @@ -209,16 +127,16 @@ impl RawCurrentUnifiedFungibleAssetBalance { } (v1_balances, v2_balances) } -} -impl From<&RawFungibleAssetBalance> for RawCurrentUnifiedFungibleAssetBalance { - fn from(fab: &RawFungibleAssetBalance) -> Self { + pub fn from_balance( + fab: &RawFungibleAssetBalance, + fa_to_coin_mapping: Option<&FungibleAssetToCoinMappings>, + ) -> Self { // Determine if this is a V2 token standard let is_v2 = matches!( TokenStandard::from_str(&fab.token_standard).expect("Invalid token standard"), TokenStandard::V2 ); - // For V2 tokens, asset_type_v2 is the original asset type // For V1 tokens, asset_type_v2 is None let asset_type_v2 = is_v2.then(|| fab.asset_type.clone()); @@ -226,9 +144,7 @@ impl From<&RawFungibleAssetBalance> for RawCurrentUnifiedFungibleAssetBalance { // For V2 tokens, look up V1 equivalent in mapping // For V1 tokens, use original asset type let asset_type_v1 = if is_v2 { - METADATA_TO_COIN_TYPE_MAPPING - .get(fab.asset_type.as_str()) - .map(|s| s.to_string()) + RawFungibleAssetToCoinMapping::get_asset_type_v1(&fab.asset_type, fa_to_coin_mapping) } else { Some(fab.asset_type.clone()) }; diff --git a/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_to_coin_mappings.rs b/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_to_coin_mappings.rs new file mode 100644 index 000000000..ef263805c --- /dev/null +++ b/rust/processor/src/db/common/models/fungible_asset_models/raw_v2_fungible_asset_to_coin_mappings.rs @@ -0,0 +1,177 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +// This is required because a diesel macro makes clippy sad +#![allow(clippy::extra_unused_lifetimes)] +#![allow(clippy::unused_unit)] + +use super::raw_v2_fungible_metadata::RawFungibleAssetMetadataModel; +use crate::{ + db::common::models::fungible_asset_models::raw_v2_fungible_asset_balances::get_paired_metadata_address, + schema::fungible_asset_to_coin_mappings, utils::database::DbPoolConnection, +}; +use ahash::AHashMap; +use diesel::query_dsl::methods::SelectDsl; +use diesel_async::RunQueryDsl; +use field_count::FieldCount; +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; + +pub type FungibleAssetToCoinMappings = AHashMap; +pub type FungibleAssetToCoinMappingsForDB = AHashMap; + +lazy_static!( + pub static ref FA_TO_COIN_MAPPING: AHashMap<&'static str, &'static str> = vec![ + ("0x000000000000000000000000000000000000000000000000000000000000000a", "0x1::aptos_coin::AptosCoin"), + ("0x377adc4848552eb2ea17259be928001923efe12271fef1667e2b784f04a7cf3a", "0x7fd500c11216f0fe3095d0c4b8aa4d64a4e2e04f83758462f2b127255643615::thl_coin::THL"), + ("0x0009da434d9b873b5159e8eeed70202ad22dc075867a7793234fbc981b63e119", "0xe4ccb6d39136469f376242c31b34d10515c8eaaa38092f804db8e08a8f53c5b2::assets_v1::EchoCoin002"), + ("0xe568e9322107a5c9ba4cbd05a630a5586aa73e744ada246c3efb0f4ce3e295f3", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDT"), + ("0xd4c0be6af89a42d78fb728dd57096eda717d7044c2dd635a01417662c33614fc", "0x55987edfab9a57f69bac759674f139ae473b5e09a9283848c1f87faf6fc1e789::shrimp::ShrimpCoin"), + ("0xa259be733b6a759909f92815927fa213904df6540519568692caf0b068fe8e62", "0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::amapt_token::AmnisApt"), + ("0xb614bfdf9edc39b330bbf9c3c5bcd0473eee2f6d4e21748629cc367869ece627", "0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::stapt_token::StakedApt"), + ("0x2b3be0a97a73c87ff62cbdd36837a9fb5bbd1d7f06a73b7ed62ec15c5326c1b8", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDC"), + ("0xb27b0c6b60772f0fc804ec1cd3339f552badf9bd1e125a7dd700d8eb11248ef1", "0x73eb84966be67e4697fc5ae75173ca6c35089e802650f75422ab49a8729704ec::coin::DooDoo"), + ("0x5915ae0eae3701833fa02e28bf530bc01ca96a5f010ac8deecb14c7a92661368", "0x4fbed3f8a3fd8a11081c8b6392152a8b0cb14d70d0414586f0c9b858fcd2d6a7::UPTOS::UPTOS"), + ("0x6dba1728c73363be1bdd4d504844c40fbb893e368ccbeff1d1bd83497dbc756d", "0xe50684a338db732d8fb8a3ac71c4b8633878bd0193bca5de2ebc852a83b35099::propbase_coin::PROPS"), + ("0xad18575b0e51dd056e1e082223c0e014cbfe4b13bc55e92f450585884f4cf951", "0x159df6b7689437016108a019fd5bef736bac692b6d4a1f10c941f6fbb9a74ca6::oft::CakeOFT"), + ("0xae02f68520afd221a5cd6fda6f5500afedab8d0a2e19a916d6d8bc2b36e758db", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WETH"), + ("0x0a9ce1bddf93b074697ec5e483bc5050bc64cff2acd31e1ccfd8ac8cae5e4abe", "0xfaf4e633ae9eb31366c9ca24214231760926576c7b625313b3688b5e900731f6::staking::StakedThalaAPT"), + ("0xd1bec63fa44dc3f3f5742c3f3a4afc3baed00505efbe955dfe6e5f9d306c67a5", "0xd0b4efb4be7c3508d9a26a9b5405cf9f860d0b9e5fe2f498b90e68b8d2cedd3e::aptos_launch_token::AptosLaunchToken"), + ("0xb81588af2f7d291e8e57f673ec74d4a38f0654633ad6bbeb1cfd4bb0550ae0df", "0x4def3d3dee27308886f0a3611dd161ce34f977a9a5de4e80b237225923492a2a::coin::T"), + ("0x02370cc1d995f3aadd337c1c6c63834ad8d2bd0cdc70bc8dff81de463e18b159", "0x53a30a6e5936c0a4c5140daed34de39d17ca7fcae08f947c02e979cef98a3719::coin::LSD"), + ("0xbe34691f0388bbca83bafab87399aeb756284e11a2872f1ae74218451cb3899f", "0xdd89c0e695df0692205912fb69fc290418bed0dbe6e4573d744a6d5e6bab6c13::coin::T"), + ("0x96d1ccca420ebc20fc8af6cacb864e44856ca879c6436d4e9be2b0a4b99bf852", "0x27fafcc4e39daac97556af8a803dbb52bcb03f0821898dc845ac54225b9793eb::move_coin::MoveCoin"), + ("0x94ed76d3d66cb0b6e7a3ab81acf830e3a50b8ae3cfb9edc0abea635a11185ff4", "0x6f986d146e4a90b828d8c12c14b6f4e003fdff11a8eecceceb63744363eaac01::mod_coin::MOD"), + ("0xa743351ed4889845737082ab9fcd42c21270647e2c6f342c509320e974035ed2", "0xd11107bdf0d6d7040c6c0bfbdecb6545191fdf13e8d8d259952f53e1713f61b5::staked_coin::StakedAptos"), + ("0x41944cf1d4dac152d692644944e2cc49ee81fafdfb37abd541d06388ec3f7eda", "0xe88ae9670071da40a9a6b1d97aab8f6f1898fdc3b8f1c1038b492dfad738448b::coin::Donk"), + ("0x4c3efb98d8d3662352f331b3465c6df263d1a7e84f002844348519614a5fea30", "0x63be1898a424616367e19bbd881f456a78470e123e2770b5b5dcdceb61279c54::movegpt_token::MovegptCoin"), + ("0x5eea0061714d46da0ccfd088df4fb1a2ea26c2421e592ade9dacc21cdb60e056", "0x1000000fa32d122c18a6a31c009ce5e71674f22d06a581bb0a15575e6addadcc::usda::USDA"), + ("0x54fc0d5fa5ad975ede1bf8b1c892ae018745a1afd4a4da9b70bb6e5448509fc0", "0x5e156f1207d0ebfa19a9eeff00d62a282278fb8719f4fab3a586a0a2c0fffbea::coin::T"), + ("0xc448a48da1ed6f6f378bb82ece996be8b5fc8dd1ea851ea2c023de17714dd747", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::UsdtCoin"), +// ("0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL"), + ("0x1912eb1a5f8f0d4c72c1687eb199b7f9d2df34da5931ec96830c557f7abaa0ad", "0x84d7aeef42d38a5ffc3ccef853e1b82e4958659d16a7de736a29c55fbbeb0114::staked_aptos_coin::StakedAptosCoin"), + ("0xb5f9a9ff6f815150af83b96b15e4f85e4e3b9e92e3fd17a414cd755c4aa49513", "0x7c0322595a73b3fc53bb166f5783470afeb1ed9f46d1176db62139991505dc61::abel_coin::AbelCoin"), + ("0x6704464238d73a679486420aab91a8a2a01feb9d700e8ba3332aa3e41d3eab62", "0xa2eda21a58856fda86451436513b867c97eecb4ba099da5775520e0f7492e852::coin::T"), + ("0xd47b65c45f5260c4f3c5de3f32ddaeabf7eab56c9493a7a955dff7f70ba8afaf", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::BusdCoin"), + ("0x92410a41654236295001f06375afbb1786dbd14bc1c42a33bfcf50204c248bb7", "0xcc8a89c8dce9693d354449f1f73e60e14e347417854f029db5bc8e7454008abb::coin::T"), + ("0x59c3a92ab1565a14d4133eb2a3418604341d37da47698a0e853c7bb22c342c55", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::DaiCoin"), + ("0x1fe81b3886ff129d42064f7ee934013de7ef968cb8f47adb5f7210192bcd4a23", "0xc26a8eda1c3ab69a157815183ddda88c89d6758ee491dd1647a70af2907ce074::coin::Chewy"), + ("0xa64d2d6f5e26daf6a3552f51d4110343b1a8c8046d0a9e72fa4086a337f3236c", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WBTC"), + ("0x2329a8351b28aa3672329217a949a9ab39d7f24534324c2eeb8b18f69e7a6fb1", "0x4ed27736e724e403f9b4645ffef0ae86fd149503f45b37c428ffabd7e46e5b05::core::RenegadeCoin"), + ("0xa79e44c5cdf8a0eb835a265a20adab56c8cdf169a2a25daa3b1d71c557b9ec59", "0xd6d6372c8bde72a7ab825c00b9edd35e643fb94a61c55d9d94a9db3010098548::USDC::Coin"), + ("0xde368b120e750bbb0d8799356ea23511306ff19edd28eed15fe7b6cc72b04388", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDD"), + ("0x41dfe1fb3d33d4d9d0b460f03ce1c0a6af6520dd8bdc0f204583c4987faf81de", "0x268d4a7a2ad93274edf6116f9f20ad8455223a7ab5fc73154f687e7dbc3e3ec6::LOON::LOON"), + ("0x4a884be56ef4c11a27162bf30b32e8e3615dcb3df0fc1777c8eb69c1991f34d0", "0xae478ff7d83ed072dbc5e264250e67ef58f57c99d89b447efd8a0a2e8b2be76e::coin::T"), + ("0xdc5e054538ba5e183d5aa197b01f327cf84aace749dc8fa2fe87bb5ec9bfe35a", "0xe1bfc010d2bdd576036f4c1f3ea7d547f19188f5b78075dd961420d843ee914c::brew_coin::BrewCoin"), + ("0xd2f2fd4a3df494042cf24c3b8c1316be8bab7ebac228be77cc0f19fcd885c666", "0x1fc2f33ab6b624e3e632ba861b755fd8e61d2c2e6cf8292e415880b4c198224d::apt20::APTS"), + ("0x9f0de082b2d4506de8b546308d4fd58bdd5ef5981097abeff40635e979874c9e", "0x65957cb717d1ec5e13c003cbad0d20d8f7f95236ea0f8bb8c419e533eda73890::TOAST::TOAST"), + ("0xc8e09f0daa8f0143318c965b43cecad55eb1a4f26ea57677fcf44c36975fe28c", "0x665d06fcd9c94430099f82973f2a5e5f13142e42fa172e72ce14f51a64bd8ad9::coin_mbx::MBX"), + ("0x5b5d60e20f3684ce19d3fd3a99ed2b2a8722b043fd67cea80ea4bc0a4749e883", "0x7c2aaaaf3f019bbf7f02beed21fc4ec352cc38272f93cb11e61ec7c89bfe5f4b::xbtc::XBTC"), + ("0x69ef9f94420a1d287892fb42450ca5777984c1c22cc886407726482480276ec1", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::WbtcCoin"), + ("0x78d37cff9f42109ce68cd73edb9ef24bb03aad697d7b6449a544701e425befbf", "0x967adbf2e05fe665ab86a3bf2c4acfa39fbf62097963474ef70a0786dae8cfa2::NRUH::NRUH"), + ("0xa36e2774e4db37934a3e27e2df7b39be5e6fcb4b7840319336fb98ffdf3d613a", "0xc82974034820c34f065f948f1bee473d481bf99fde2d23e981043e5038cb36be::WOOF::Woof"), + ("0x37bdfd533a28c38ba6f2963e3f2ab881b3826d952ea3d4ca03020e0d2735348d", "0xccc9620d38c4f3991fa68a03ad98ef3735f18d04717cb75d7a1300dd8a7eed75::coin::T"), + ("0xeb73df9d3ba3fbc2538d2e7f4a2dac9718b48b07f65596a9c7cc84d978e3d6cd", "0x2dcbc03740a6fa2efee926b9df329184cce357d0573bdab09930f4d48e61a4c8::STOS::STOS"), + ("0xcd70630fb90cab716ab01a7884821f86dceb1bbb09a89683b5c22c5462503f51", "0xdcfa079344261bfde45e7f6281df091743b8d3098bf9e26e1c0212fc5b070621::zaaptos_token::ZaaptosCoin"), + ("0x1b976ce1a6bf4a37057166f52646c879f37cb0712eb2fd4005e54b9c929b7171", "0x380b3422c7d2c28f29776cf1a234b98bc514b6d8c1cf16e1b9123cb1acb6203::plant_coin::PlantCoin"), + ("0xfd1f22b455e2d095a1ec497ecc0a11db86173d0271557ca6c16baa9d66466a75", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x1::aptos_coin::AptosCoin, 0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDC>"), + ("0x0943bf6e5329167f459a4ae7efa93336f49ef08a5aff65f8c70133f0a0725ef2", "0x91b54cb4441c88fa21b7ca5b8b860e8b6fe726c23866bed91999823e65c1026d::GEMKRW::GEMKRW"), + ("0xf92047adba5ec4a21ad076b19a0c8806b195435696d30dc3f43781d1e6d91563", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::UsdcCoin"), + ("0xf0876baf6f8c37723f0e9d9c1bbad1ccb49324c228bcc906e2f1f5a9e139eda1", "0xcfea864b32833f157f042618bd845145256b1bf4c0da34a7013b76e42daa53cc::usdy::USDY"), + ("0xf7833d21f83a19548c81e8fd17d8bde4a6e8cc3fbb1ffb97973e06e261c75dee", "0x5c738a5dfa343bee927c39ebe85b0ceb95fdb5ee5b323c95559614f5a77c47cf::Aptoge::Aptoge"), + ("0xa4607412abfc37ec0b6fd6e102f5f0b7989f59fd44ff5d374cbe360ffbecdfff", "0xf891d2e004973430cc2bbbee69f3d0f4adb9c7ae03137b4579f7bb9979283ee6::APTOS_FOMO::APTOS_FOMO"), + ("0x109492c6323a413d605f5768127b11ef28d5805a818b355b0c9ebcb1995fcf81", "0x881ac202b1f1e6ad4efcff7a1d0579411533f2502417a19211cfc49751ddb5f4::coin::MOJO"), + ("0xfbd6406c12cab2aef728c917a365cdb73883213f74af5e8a46c8fbd77b623ee7", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::WethCoin"), + ("0xc40443d625f94ddec95a76bcf2534eda394bf67713b93f08eb202026e2aaa66a", "0xdf3d5eb83df80dfde8ceb1edaa24d8dbc46da6a89ae134a858338e1b86a29e38::coin::Returd"), + ("0x08bbc1e07f934be0862be6df1477dbab54d6ccdf594f1569a64fa2941cbfe368", "0x198e4a77b72cbcac7465e774e99d2ba552053ca57b0759ea3c008433742b4e4f::PEPE::Pepe"), + ("0x290c792f89a47cd7280e0b9035fa8b2876ab4298f0135d4a2c88e77257681ea1", "0x2778b277644d375721766abfff0df2adca795f6cbae9f02ff1c95ce9adb6ee28::aptos_shiba_coin::AptosShibaCoin"), + ("0xc73b3f454576b00d4d05393ff427537eda42f791350f30ce1f566448b5798644", "0x66302f3c648890f70ca3fafc42c919483945f3ba155101bc2e149e38a8b93afc::toss_coin::TossCoin"), + ("0x3fb0cd30095fc85c77eb5cb9edcdbead3cef34e449b1a6f07729282120bc6383", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::BnbCoin"), + ("0xa0d9d647c5737a5aed08d2cfeb39c31cf901d44bc4aa024eaa7e5e68b804e011", "0xfaf4e633ae9eb31366c9ca24214231760926576c7b625313b3688b5e900731f6::staking::ThalaAPT"), + ("0x0dcee4819a7b45113c6e44a157a11866f3366a7c93f79ba5acdf27f6fb8ce301", "0xbe3c4b493632b4d776d56e19d91dcfb34f591f759f6b57f8632d455360da503c::dumdum_coin::DumdumCoin"), + ("0x23813a24e98215ab541051432b734baecaa3737019a4891c37034f88d9944960", "0x407a220699982ebb514568d007938d2447d33667e4418372ffec1ddb24491b6c::coin::T"), + ("0xb56df862320ff2dc317e147c870ad09f12711a5e02c6245438f827e6c54188b4", "0x5c738a5dfa343bee927c39ebe85b0ceb95fdb5ee5b323c95559614f5a77c47cf::AptSwap::AptSwapGovernance"), + ("0xf5d23515c4454652c38219aec5f1a0720207dc1f5d2e5140b94608f9ce821a36", "0x84edd115c901709ef28f3cb66a82264ba91bfd24789500b6fd34ab9e8888e272::coin::DLC"), + ("0xcc71496adf8086dcc62bbe4d975718f09de8cc42629aacd8df84df514cbac154", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x1::aptos_coin::AptosCoin, 0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDT>"), + ("0xfad230e7d9df2baf83a68b6f50217ed3c06da593e766970a885965b43b894a04", "0xada35ada7e43e2ee1c39633ffccec38b76ce702b4efc2e60b50f63fbe4f710d8::apetos_token::ApetosCoin"), + ("0x8d55e255fcf4142a8008b1b5ca0ec9efb964f8a914807c0bf279f6c3af3ef955", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::amapt_token::AmnisApt, 0x1::aptos_coin::AptosCoin>"), + ("0x66037303c053e2bd0d5af444a9a4792e5a2b56633849e57259e575349d871d04", "0x1eb35b3b9e911ca2093970ae5abfb970dbec54597d43954bb53b09d2e8356cc3::BMTAmm::BMTAmm"), + ("0xcab64ed0d956462e9b8ba7c340fdb8b9ab52da1503f37b522288bc0c5bf944de", "0x389dbbc6884a1d5b1ab4e1df2913a8c1b01251e50aed377554372b2842c5e3ef::EONcoin::EONCoin"), + ("0x5486d29c4fceec48c55e88a700eddfdf5be8663a2a873ac0d2baac21cd78b390", "0xacd014e8bdf395fa8497b6d585b164547a9d45269377bdf67c96c541b7fec9ed::coin::T"), + ("0x9660042a7c01d776938184278381d24c7009ca385d9a59cf9b22691f97615960", "0x16fe2df00ea7dde4a63409201f7f4e536bde7bb7335526a35d05111e68aa322c::AnimeCoin::ANI"), + ("0x4ddd6e6dfff083e2e4981cf959384e6aec18a9c62cc4694e8aab950c07296208", "0xcc78307c77f1c2c0fdfee17269bfca7876a0b35438c3442417480c0d5c370fbc::AptopadCoin::APD"), + ("0x8ddd74585cfe3249d71532a8c96fbfc3a1d1134357ec1eaf0411db953c6b942b", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WETH, 0x1::aptos_coin::AptosCoin>"), + ("0x70003a36f53ed8142a7a530415982d466bc16cdcf2be2599c08211105bd9364d", "0x26f03cd414cdcae387961058ab5523a4e64559f4b0853c46c9b3c4e01fd8af9b::usdy_staging::USDY_STAGING"), + ("0x64196428c8f492b189a1f69234e12b161adf729fc42679cb30219a59f8114f8c", "0x26f03cd414cdcae387961058ab5523a4e64559f4b0853c46c9b3c4e01fd8af9b::usdy::USDY"), + ("0x56998a39007a9b431fa9d98bcc57de14f2f357846723a1ad94dfb4c3c965a3a3", "0x14b0ef0ec69f346bea3dfa0c5a8c3942fb05c08760059948f9f24c02cd0d4fd8::mover_token::Mover"), +].iter().cloned().collect(); + ); + +#[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)] +#[diesel(primary_key(coin_type))] +#[diesel(table_name = fungible_asset_to_coin_mappings)] +pub struct RawFungibleAssetToCoinMapping { + pub fungible_asset_metadata_address: String, + pub coin_type: String, + pub last_transaction_version: i64, +} + +impl RawFungibleAssetToCoinMapping { + pub fn from_raw_fungible_asset_metadata(metadata: &RawFungibleAssetMetadataModel) -> Self { + let fungible_asset_metadata_address = get_paired_metadata_address(&metadata.asset_type); + Self { + fungible_asset_metadata_address, + coin_type: metadata.asset_type.clone(), + last_transaction_version: metadata.last_transaction_version, + } + } + + /// Get the asset_type_v1 (coin type) from either the constant or the dynamic mapping + pub fn get_asset_type_v1( + asset_type_v2: &str, + fa_to_coin_mapping: Option<&FungibleAssetToCoinMappings>, + ) -> Option { + // First check if asset type exists in static mapping + if let Some(&coin_type) = FA_TO_COIN_MAPPING.get(asset_type_v2) { + return Some(coin_type.to_string()); + } + // If not found in static mapping, check dynamic mapping if provided + fa_to_coin_mapping.and_then(|mapping| mapping.get(asset_type_v2).cloned()) + } + + /// This should be triggered on startup only. After that we will use the mapping in memory. + pub async fn get_all_mappings(conn: &mut DbPoolConnection<'_>) -> FungibleAssetToCoinMappings { + match Self::query_all(conn).await { + Ok(mappings) => { + let mut result = AHashMap::new(); + for (metadata_address, coin_type) in mappings { + result.insert(metadata_address, coin_type); + } + result + }, + Err(e) => { + tracing::error!( + error = ?e, + "Failed to query fungible asset to coin mappings" + ); + panic!("Failed to query fungible asset to coin mappings: {:?}", e); + }, + } + } + + async fn query_all( + conn: &mut DbPoolConnection<'_>, + ) -> diesel::QueryResult> { + fungible_asset_to_coin_mappings::table + .select(( + fungible_asset_to_coin_mappings::fungible_asset_metadata_address, + fungible_asset_to_coin_mappings::coin_type, + )) + .load::<(String, String)>(conn) + .await + } +} + +pub trait FungibleAssetToCoinMappingConvertible { + fn from_raw(raw: RawFungibleAssetToCoinMapping) -> Self; +} diff --git a/rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/down.sql b/rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/down.sql new file mode 100644 index 000000000..3fcb50751 --- /dev/null +++ b/rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/down.sql @@ -0,0 +1,4 @@ +-- Drop indices first +DROP INDEX IF EXISTS fatcm_coin_type_idx; +-- Drop the table +DROP TABLE IF EXISTS fungible_asset_to_coin_mappings; \ No newline at end of file diff --git a/rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/up.sql b/rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/up.sql new file mode 100644 index 000000000..8eec2dd1c --- /dev/null +++ b/rust/processor/src/db/postgres/migrations/2025-01-23-164631_coin_infos_for_migration/up.sql @@ -0,0 +1,8 @@ +-- Create coin to fungible asset mapping table for migration +CREATE TABLE IF NOT EXISTS fungible_asset_to_coin_mappings ( + fungible_asset_metadata_address VARCHAR(66) NOT NULL PRIMARY KEY, + coin_type VARCHAR(1000) NOT NULL, + last_transaction_version BIGINT NOT NULL +); +-- Create indices +CREATE INDEX IF NOT EXISTS fatcm_coin_type_idx ON fungible_asset_to_coin_mappings (coin_type); \ No newline at end of file diff --git a/rust/processor/src/db/postgres/models/fungible_asset_models/mod.rs b/rust/processor/src/db/postgres/models/fungible_asset_models/mod.rs index a22e0bc14..49388d77c 100644 --- a/rust/processor/src/db/postgres/models/fungible_asset_models/mod.rs +++ b/rust/processor/src/db/postgres/models/fungible_asset_models/mod.rs @@ -3,6 +3,7 @@ pub mod v2_fungible_asset_activities; pub mod v2_fungible_asset_balances; +pub mod v2_fungible_asset_to_coin_mappings; pub mod v2_fungible_asset_utils; pub mod v2_fungible_metadata; diff --git a/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_balances.rs b/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_balances.rs index f8baad6db..fa4bc88ea 100644 --- a/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_balances.rs +++ b/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_balances.rs @@ -15,99 +15,10 @@ use crate::{ fungible_asset_balances, }, }; -use ahash::AHashMap; use bigdecimal::BigDecimal; use field_count::FieldCount; -use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; -lazy_static!( - pub static ref METADATA_TO_COIN_TYPE_MAPPING: AHashMap<&'static str, &'static str> = vec![ - ("0x000000000000000000000000000000000000000000000000000000000000000a", "0x1::aptos_coin::AptosCoin"), - ("0x377adc4848552eb2ea17259be928001923efe12271fef1667e2b784f04a7cf3a", "0x7fd500c11216f0fe3095d0c4b8aa4d64a4e2e04f83758462f2b127255643615::thl_coin::THL"), - ("0x0009da434d9b873b5159e8eeed70202ad22dc075867a7793234fbc981b63e119", "0xe4ccb6d39136469f376242c31b34d10515c8eaaa38092f804db8e08a8f53c5b2::assets_v1::EchoCoin002"), - ("0xe568e9322107a5c9ba4cbd05a630a5586aa73e744ada246c3efb0f4ce3e295f3", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDT"), - ("0xd4c0be6af89a42d78fb728dd57096eda717d7044c2dd635a01417662c33614fc", "0x55987edfab9a57f69bac759674f139ae473b5e09a9283848c1f87faf6fc1e789::shrimp::ShrimpCoin"), - ("0xa259be733b6a759909f92815927fa213904df6540519568692caf0b068fe8e62", "0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::amapt_token::AmnisApt"), - ("0xb614bfdf9edc39b330bbf9c3c5bcd0473eee2f6d4e21748629cc367869ece627", "0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::stapt_token::StakedApt"), - ("0x2b3be0a97a73c87ff62cbdd36837a9fb5bbd1d7f06a73b7ed62ec15c5326c1b8", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDC"), - ("0xb27b0c6b60772f0fc804ec1cd3339f552badf9bd1e125a7dd700d8eb11248ef1", "0x73eb84966be67e4697fc5ae75173ca6c35089e802650f75422ab49a8729704ec::coin::DooDoo"), - ("0x5915ae0eae3701833fa02e28bf530bc01ca96a5f010ac8deecb14c7a92661368", "0x4fbed3f8a3fd8a11081c8b6392152a8b0cb14d70d0414586f0c9b858fcd2d6a7::UPTOS::UPTOS"), - ("0x6dba1728c73363be1bdd4d504844c40fbb893e368ccbeff1d1bd83497dbc756d", "0xe50684a338db732d8fb8a3ac71c4b8633878bd0193bca5de2ebc852a83b35099::propbase_coin::PROPS"), - ("0xad18575b0e51dd056e1e082223c0e014cbfe4b13bc55e92f450585884f4cf951", "0x159df6b7689437016108a019fd5bef736bac692b6d4a1f10c941f6fbb9a74ca6::oft::CakeOFT"), - ("0xae02f68520afd221a5cd6fda6f5500afedab8d0a2e19a916d6d8bc2b36e758db", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WETH"), - ("0x0a9ce1bddf93b074697ec5e483bc5050bc64cff2acd31e1ccfd8ac8cae5e4abe", "0xfaf4e633ae9eb31366c9ca24214231760926576c7b625313b3688b5e900731f6::staking::StakedThalaAPT"), - ("0xd1bec63fa44dc3f3f5742c3f3a4afc3baed00505efbe955dfe6e5f9d306c67a5", "0xd0b4efb4be7c3508d9a26a9b5405cf9f860d0b9e5fe2f498b90e68b8d2cedd3e::aptos_launch_token::AptosLaunchToken"), - ("0xb81588af2f7d291e8e57f673ec74d4a38f0654633ad6bbeb1cfd4bb0550ae0df", "0x4def3d3dee27308886f0a3611dd161ce34f977a9a5de4e80b237225923492a2a::coin::T"), - ("0x02370cc1d995f3aadd337c1c6c63834ad8d2bd0cdc70bc8dff81de463e18b159", "0x53a30a6e5936c0a4c5140daed34de39d17ca7fcae08f947c02e979cef98a3719::coin::LSD"), - ("0xbe34691f0388bbca83bafab87399aeb756284e11a2872f1ae74218451cb3899f", "0xdd89c0e695df0692205912fb69fc290418bed0dbe6e4573d744a6d5e6bab6c13::coin::T"), - ("0x96d1ccca420ebc20fc8af6cacb864e44856ca879c6436d4e9be2b0a4b99bf852", "0x27fafcc4e39daac97556af8a803dbb52bcb03f0821898dc845ac54225b9793eb::move_coin::MoveCoin"), - ("0x94ed76d3d66cb0b6e7a3ab81acf830e3a50b8ae3cfb9edc0abea635a11185ff4", "0x6f986d146e4a90b828d8c12c14b6f4e003fdff11a8eecceceb63744363eaac01::mod_coin::MOD"), - ("0xa743351ed4889845737082ab9fcd42c21270647e2c6f342c509320e974035ed2", "0xd11107bdf0d6d7040c6c0bfbdecb6545191fdf13e8d8d259952f53e1713f61b5::staked_coin::StakedAptos"), - ("0x41944cf1d4dac152d692644944e2cc49ee81fafdfb37abd541d06388ec3f7eda", "0xe88ae9670071da40a9a6b1d97aab8f6f1898fdc3b8f1c1038b492dfad738448b::coin::Donk"), - ("0x4c3efb98d8d3662352f331b3465c6df263d1a7e84f002844348519614a5fea30", "0x63be1898a424616367e19bbd881f456a78470e123e2770b5b5dcdceb61279c54::movegpt_token::MovegptCoin"), - ("0x5eea0061714d46da0ccfd088df4fb1a2ea26c2421e592ade9dacc21cdb60e056", "0x1000000fa32d122c18a6a31c009ce5e71674f22d06a581bb0a15575e6addadcc::usda::USDA"), - ("0x54fc0d5fa5ad975ede1bf8b1c892ae018745a1afd4a4da9b70bb6e5448509fc0", "0x5e156f1207d0ebfa19a9eeff00d62a282278fb8719f4fab3a586a0a2c0fffbea::coin::T"), - ("0xc448a48da1ed6f6f378bb82ece996be8b5fc8dd1ea851ea2c023de17714dd747", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::UsdtCoin"), - ("0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL"), - ("0x1912eb1a5f8f0d4c72c1687eb199b7f9d2df34da5931ec96830c557f7abaa0ad", "0x84d7aeef42d38a5ffc3ccef853e1b82e4958659d16a7de736a29c55fbbeb0114::staked_aptos_coin::StakedAptosCoin"), - ("0xb5f9a9ff6f815150af83b96b15e4f85e4e3b9e92e3fd17a414cd755c4aa49513", "0x7c0322595a73b3fc53bb166f5783470afeb1ed9f46d1176db62139991505dc61::abel_coin::AbelCoin"), - ("0x6704464238d73a679486420aab91a8a2a01feb9d700e8ba3332aa3e41d3eab62", "0xa2eda21a58856fda86451436513b867c97eecb4ba099da5775520e0f7492e852::coin::T"), - ("0xd47b65c45f5260c4f3c5de3f32ddaeabf7eab56c9493a7a955dff7f70ba8afaf", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::BusdCoin"), - ("0x92410a41654236295001f06375afbb1786dbd14bc1c42a33bfcf50204c248bb7", "0xcc8a89c8dce9693d354449f1f73e60e14e347417854f029db5bc8e7454008abb::coin::T"), - ("0x59c3a92ab1565a14d4133eb2a3418604341d37da47698a0e853c7bb22c342c55", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::DaiCoin"), - ("0x1fe81b3886ff129d42064f7ee934013de7ef968cb8f47adb5f7210192bcd4a23", "0xc26a8eda1c3ab69a157815183ddda88c89d6758ee491dd1647a70af2907ce074::coin::Chewy"), - ("0xa64d2d6f5e26daf6a3552f51d4110343b1a8c8046d0a9e72fa4086a337f3236c", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WBTC"), - ("0x2329a8351b28aa3672329217a949a9ab39d7f24534324c2eeb8b18f69e7a6fb1", "0x4ed27736e724e403f9b4645ffef0ae86fd149503f45b37c428ffabd7e46e5b05::core::RenegadeCoin"), - ("0xa79e44c5cdf8a0eb835a265a20adab56c8cdf169a2a25daa3b1d71c557b9ec59", "0xd6d6372c8bde72a7ab825c00b9edd35e643fb94a61c55d9d94a9db3010098548::USDC::Coin"), - ("0xde368b120e750bbb0d8799356ea23511306ff19edd28eed15fe7b6cc72b04388", "0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDD"), - ("0x41dfe1fb3d33d4d9d0b460f03ce1c0a6af6520dd8bdc0f204583c4987faf81de", "0x268d4a7a2ad93274edf6116f9f20ad8455223a7ab5fc73154f687e7dbc3e3ec6::LOON::LOON"), - ("0x4a884be56ef4c11a27162bf30b32e8e3615dcb3df0fc1777c8eb69c1991f34d0", "0xae478ff7d83ed072dbc5e264250e67ef58f57c99d89b447efd8a0a2e8b2be76e::coin::T"), - ("0xdc5e054538ba5e183d5aa197b01f327cf84aace749dc8fa2fe87bb5ec9bfe35a", "0xe1bfc010d2bdd576036f4c1f3ea7d547f19188f5b78075dd961420d843ee914c::brew_coin::BrewCoin"), - ("0xd2f2fd4a3df494042cf24c3b8c1316be8bab7ebac228be77cc0f19fcd885c666", "0x1fc2f33ab6b624e3e632ba861b755fd8e61d2c2e6cf8292e415880b4c198224d::apt20::APTS"), - ("0x9f0de082b2d4506de8b546308d4fd58bdd5ef5981097abeff40635e979874c9e", "0x65957cb717d1ec5e13c003cbad0d20d8f7f95236ea0f8bb8c419e533eda73890::TOAST::TOAST"), - ("0xc8e09f0daa8f0143318c965b43cecad55eb1a4f26ea57677fcf44c36975fe28c", "0x665d06fcd9c94430099f82973f2a5e5f13142e42fa172e72ce14f51a64bd8ad9::coin_mbx::MBX"), - ("0x5b5d60e20f3684ce19d3fd3a99ed2b2a8722b043fd67cea80ea4bc0a4749e883", "0x7c2aaaaf3f019bbf7f02beed21fc4ec352cc38272f93cb11e61ec7c89bfe5f4b::xbtc::XBTC"), - ("0x69ef9f94420a1d287892fb42450ca5777984c1c22cc886407726482480276ec1", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::WbtcCoin"), - ("0x78d37cff9f42109ce68cd73edb9ef24bb03aad697d7b6449a544701e425befbf", "0x967adbf2e05fe665ab86a3bf2c4acfa39fbf62097963474ef70a0786dae8cfa2::NRUH::NRUH"), - ("0xa36e2774e4db37934a3e27e2df7b39be5e6fcb4b7840319336fb98ffdf3d613a", "0xc82974034820c34f065f948f1bee473d481bf99fde2d23e981043e5038cb36be::WOOF::Woof"), - ("0x37bdfd533a28c38ba6f2963e3f2ab881b3826d952ea3d4ca03020e0d2735348d", "0xccc9620d38c4f3991fa68a03ad98ef3735f18d04717cb75d7a1300dd8a7eed75::coin::T"), - ("0xeb73df9d3ba3fbc2538d2e7f4a2dac9718b48b07f65596a9c7cc84d978e3d6cd", "0x2dcbc03740a6fa2efee926b9df329184cce357d0573bdab09930f4d48e61a4c8::STOS::STOS"), - ("0xcd70630fb90cab716ab01a7884821f86dceb1bbb09a89683b5c22c5462503f51", "0xdcfa079344261bfde45e7f6281df091743b8d3098bf9e26e1c0212fc5b070621::zaaptos_token::ZaaptosCoin"), - ("0x1b976ce1a6bf4a37057166f52646c879f37cb0712eb2fd4005e54b9c929b7171", "0x380b3422c7d2c28f29776cf1a234b98bc514b6d8c1cf16e1b9123cb1acb6203::plant_coin::PlantCoin"), - ("0xfd1f22b455e2d095a1ec497ecc0a11db86173d0271557ca6c16baa9d66466a75", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x1::aptos_coin::AptosCoin, 0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDC>"), - ("0x0943bf6e5329167f459a4ae7efa93336f49ef08a5aff65f8c70133f0a0725ef2", "0x91b54cb4441c88fa21b7ca5b8b860e8b6fe726c23866bed91999823e65c1026d::GEMKRW::GEMKRW"), - ("0xf92047adba5ec4a21ad076b19a0c8806b195435696d30dc3f43781d1e6d91563", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::UsdcCoin"), - ("0xf0876baf6f8c37723f0e9d9c1bbad1ccb49324c228bcc906e2f1f5a9e139eda1", "0xcfea864b32833f157f042618bd845145256b1bf4c0da34a7013b76e42daa53cc::usdy::USDY"), - ("0xf7833d21f83a19548c81e8fd17d8bde4a6e8cc3fbb1ffb97973e06e261c75dee", "0x5c738a5dfa343bee927c39ebe85b0ceb95fdb5ee5b323c95559614f5a77c47cf::Aptoge::Aptoge"), - ("0xa4607412abfc37ec0b6fd6e102f5f0b7989f59fd44ff5d374cbe360ffbecdfff", "0xf891d2e004973430cc2bbbee69f3d0f4adb9c7ae03137b4579f7bb9979283ee6::APTOS_FOMO::APTOS_FOMO"), - ("0x109492c6323a413d605f5768127b11ef28d5805a818b355b0c9ebcb1995fcf81", "0x881ac202b1f1e6ad4efcff7a1d0579411533f2502417a19211cfc49751ddb5f4::coin::MOJO"), - ("0xfbd6406c12cab2aef728c917a365cdb73883213f74af5e8a46c8fbd77b623ee7", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::WethCoin"), - ("0xc40443d625f94ddec95a76bcf2534eda394bf67713b93f08eb202026e2aaa66a", "0xdf3d5eb83df80dfde8ceb1edaa24d8dbc46da6a89ae134a858338e1b86a29e38::coin::Returd"), - ("0x08bbc1e07f934be0862be6df1477dbab54d6ccdf594f1569a64fa2941cbfe368", "0x198e4a77b72cbcac7465e774e99d2ba552053ca57b0759ea3c008433742b4e4f::PEPE::Pepe"), - ("0x290c792f89a47cd7280e0b9035fa8b2876ab4298f0135d4a2c88e77257681ea1", "0x2778b277644d375721766abfff0df2adca795f6cbae9f02ff1c95ce9adb6ee28::aptos_shiba_coin::AptosShibaCoin"), - ("0xc73b3f454576b00d4d05393ff427537eda42f791350f30ce1f566448b5798644", "0x66302f3c648890f70ca3fafc42c919483945f3ba155101bc2e149e38a8b93afc::toss_coin::TossCoin"), - ("0x3fb0cd30095fc85c77eb5cb9edcdbead3cef34e449b1a6f07729282120bc6383", "0x8d87a65ba30e09357fa2edea2c80dbac296e5dec2b18287113500b902942929d::celer_coin_manager::BnbCoin"), - ("0xa0d9d647c5737a5aed08d2cfeb39c31cf901d44bc4aa024eaa7e5e68b804e011", "0xfaf4e633ae9eb31366c9ca24214231760926576c7b625313b3688b5e900731f6::staking::ThalaAPT"), - ("0x0dcee4819a7b45113c6e44a157a11866f3366a7c93f79ba5acdf27f6fb8ce301", "0xbe3c4b493632b4d776d56e19d91dcfb34f591f759f6b57f8632d455360da503c::dumdum_coin::DumdumCoin"), - ("0x23813a24e98215ab541051432b734baecaa3737019a4891c37034f88d9944960", "0x407a220699982ebb514568d007938d2447d33667e4418372ffec1ddb24491b6c::coin::T"), - ("0xb56df862320ff2dc317e147c870ad09f12711a5e02c6245438f827e6c54188b4", "0x5c738a5dfa343bee927c39ebe85b0ceb95fdb5ee5b323c95559614f5a77c47cf::AptSwap::AptSwapGovernance"), - ("0xf5d23515c4454652c38219aec5f1a0720207dc1f5d2e5140b94608f9ce821a36", "0x84edd115c901709ef28f3cb66a82264ba91bfd24789500b6fd34ab9e8888e272::coin::DLC"), - ("0xcc71496adf8086dcc62bbe4d975718f09de8cc42629aacd8df84df514cbac154", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x1::aptos_coin::AptosCoin, 0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::USDT>"), - ("0xfad230e7d9df2baf83a68b6f50217ed3c06da593e766970a885965b43b894a04", "0xada35ada7e43e2ee1c39633ffccec38b76ce702b4efc2e60b50f63fbe4f710d8::apetos_token::ApetosCoin"), - ("0x8d55e255fcf4142a8008b1b5ca0ec9efb964f8a914807c0bf279f6c3af3ef955", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0x111ae3e5bc816a5e63c2da97d0aa3886519e0cd5e4b046659fa35796bd11542a::amapt_token::AmnisApt, 0x1::aptos_coin::AptosCoin>"), - ("0x66037303c053e2bd0d5af444a9a4792e5a2b56633849e57259e575349d871d04", "0x1eb35b3b9e911ca2093970ae5abfb970dbec54597d43954bb53b09d2e8356cc3::BMTAmm::BMTAmm"), - ("0xcab64ed0d956462e9b8ba7c340fdb8b9ab52da1503f37b522288bc0c5bf944de", "0x389dbbc6884a1d5b1ab4e1df2913a8c1b01251e50aed377554372b2842c5e3ef::EONcoin::EONCoin"), - ("0x5486d29c4fceec48c55e88a700eddfdf5be8663a2a873ac0d2baac21cd78b390", "0xacd014e8bdf395fa8497b6d585b164547a9d45269377bdf67c96c541b7fec9ed::coin::T"), - ("0x9660042a7c01d776938184278381d24c7009ca385d9a59cf9b22691f97615960", "0x16fe2df00ea7dde4a63409201f7f4e536bde7bb7335526a35d05111e68aa322c::AnimeCoin::ANI"), - ("0x4ddd6e6dfff083e2e4981cf959384e6aec18a9c62cc4694e8aab950c07296208", "0xcc78307c77f1c2c0fdfee17269bfca7876a0b35438c3442417480c0d5c370fbc::AptopadCoin::APD"), - ("0x8ddd74585cfe3249d71532a8c96fbfc3a1d1134357ec1eaf0411db953c6b942b", "0x31a6675cbe84365bf2b0cbce617ece6c47023ef70826533bde5203d32171dc3c::swap::LPToken<0xf22bede237a07e121b56d91a491eb7bcdfd1f5907926a9e58338f964a01b17fa::asset::WETH, 0x1::aptos_coin::AptosCoin>"), - ("0x70003a36f53ed8142a7a530415982d466bc16cdcf2be2599c08211105bd9364d", "0x26f03cd414cdcae387961058ab5523a4e64559f4b0853c46c9b3c4e01fd8af9b::usdy_staging::USDY_STAGING"), - ("0x64196428c8f492b189a1f69234e12b161adf729fc42679cb30219a59f8114f8c", "0x26f03cd414cdcae387961058ab5523a4e64559f4b0853c46c9b3c4e01fd8af9b::usdy::USDY"), - ("0x56998a39007a9b431fa9d98bcc57de14f2f357846723a1ad94dfb4c3c965a3a3", "0x14b0ef0ec69f346bea3dfa0c5a8c3942fb05c08760059948f9f24c02cd0d4fd8::mover_token::Mover"), -].iter().cloned().collect(); - ); - #[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)] #[diesel(primary_key(transaction_version, write_set_change_index))] #[diesel(table_name = fungible_asset_balances)] diff --git a/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_to_coin_mappings.rs b/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_to_coin_mappings.rs new file mode 100644 index 000000000..1bdfd261b --- /dev/null +++ b/rust/processor/src/db/postgres/models/fungible_asset_models/v2_fungible_asset_to_coin_mappings.rs @@ -0,0 +1,34 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +// This is required because a diesel macro makes clippy sad +#![allow(clippy::extra_unused_lifetimes)] +#![allow(clippy::unused_unit)] + +use crate::{ + db::common::models::fungible_asset_models::raw_v2_fungible_asset_to_coin_mappings::{ + FungibleAssetToCoinMappingConvertible, RawFungibleAssetToCoinMapping, + }, + schema::fungible_asset_to_coin_mappings, +}; +use field_count::FieldCount; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, FieldCount, Identifiable, Insertable, Serialize)] +#[diesel(primary_key(coin_type))] +#[diesel(table_name = fungible_asset_to_coin_mappings)] +pub struct FungibleAssetToCoinMapping { + pub fungible_asset_metadata_address: String, + pub coin_type: String, + pub last_transaction_version: i64, +} + +impl FungibleAssetToCoinMappingConvertible for FungibleAssetToCoinMapping { + fn from_raw(raw: RawFungibleAssetToCoinMapping) -> Self { + Self { + fungible_asset_metadata_address: raw.fungible_asset_metadata_address, + coin_type: raw.coin_type, + last_transaction_version: raw.last_transaction_version, + } + } +} diff --git a/rust/processor/src/db/postgres/schema.rs b/rust/processor/src/db/postgres/schema.rs index 9f7f46ef8..dfad031b8 100644 --- a/rust/processor/src/db/postgres/schema.rs +++ b/rust/processor/src/db/postgres/schema.rs @@ -881,6 +881,16 @@ diesel::table! { } } +diesel::table! { + fungible_asset_to_coin_mappings (fungible_asset_metadata_address) { + #[max_length = 66] + fungible_asset_metadata_address -> Varchar, + #[max_length = 1000] + coin_type -> Varchar, + last_transaction_version -> Int8, + } +} + diesel::table! { indexer_status (db) { #[max_length = 50] @@ -1382,6 +1392,7 @@ diesel::allow_tables_to_appear_in_same_query!( fungible_asset_activities, fungible_asset_balances, fungible_asset_metadata, + fungible_asset_to_coin_mappings, indexer_status, ledger_infos, move_modules, diff --git a/rust/processor/src/processors/fungible_asset_processor.rs b/rust/processor/src/processors/fungible_asset_processor.rs index 1ff563e27..25027a004 100644 --- a/rust/processor/src/processors/fungible_asset_processor.rs +++ b/rust/processor/src/processors/fungible_asset_processor.rs @@ -13,6 +13,10 @@ use crate::{ CurrentUnifiedFungibleAssetBalanceConvertible, RawCurrentUnifiedFungibleAssetBalance, RawFungibleAssetBalance, }, + raw_v2_fungible_asset_to_coin_mappings::{ + FungibleAssetToCoinMappingConvertible, FungibleAssetToCoinMappings, + FungibleAssetToCoinMappingsForDB, RawFungibleAssetToCoinMapping, + }, raw_v2_fungible_metadata::{ FungibleAssetMetadataConvertible, FungibleAssetMetadataMapping, RawFungibleAssetMetadataModel, @@ -29,6 +33,7 @@ use crate::{ v2_fungible_asset_balances::{ CurrentUnifiedFungibleAssetBalance, FungibleAssetBalance, }, + v2_fungible_asset_to_coin_mappings::FungibleAssetToCoinMapping, v2_fungible_asset_utils::FeeStatement, v2_fungible_metadata::FungibleAssetMetadataModel, }, @@ -103,6 +108,7 @@ async fn insert_to_db( &[CurrentUnifiedFungibleAssetBalance], ), coin_supply: &[CoinSupply], + fungible_asset_to_coin_mappings: &[FungibleAssetToCoinMapping], per_table_chunk_sizes: &AHashMap, ) -> Result<(), diesel::result::Error> { tracing::trace!( @@ -149,14 +155,23 @@ async fn insert_to_db( ), ); let cs = execute_in_chunks( - conn, + conn.clone(), insert_coin_supply_query, coin_supply, get_config_table_chunk_size::("coin_supply", per_table_chunk_sizes), ); - let (faa_res, fam_res, cufab1_res, cufab2_res, cs_res) = - tokio::join!(faa, fam, cufab_v1, cufab_v2, cs); - for res in [faa_res, fam_res, cufab1_res, cufab2_res, cs_res] { + let fatcm = execute_in_chunks( + conn, + insert_fungible_asset_to_coin_mappings_query, + fungible_asset_to_coin_mappings, + get_config_table_chunk_size::( + "fungible_asset_to_coin_mappings", + per_table_chunk_sizes, + ), + ); + let (faa_res, fam_res, cufab1_res, cufab2_res, cs_res, fatcm_res) = + tokio::join!(faa, fam, cufab_v1, cufab_v2, cs, fatcm); + for res in [faa_res, fam_res, cufab1_res, cufab2_res, cs_res, fatcm_res] { res?; } @@ -311,6 +326,27 @@ pub fn insert_coin_supply_query( ) } +pub fn insert_fungible_asset_to_coin_mappings_query( + items_to_insert: Vec, +) -> ( + impl QueryFragment + diesel::query_builder::QueryId + Send, + Option<&'static str>, +) { + use schema::fungible_asset_to_coin_mappings::dsl::*; + + ( + diesel::insert_into(schema::fungible_asset_to_coin_mappings::table) + .values(items_to_insert) + .on_conflict(fungible_asset_metadata_address) + .do_update() + .set(( + coin_type.eq(excluded(coin_type)), + last_transaction_version.eq(excluded(last_transaction_version)), + )), + Some(" WHERE fungible_asset_to_coin_mappings.last_transaction_version <= excluded.last_transaction_version "), + ) +} + #[async_trait] impl ProcessorTrait for FungibleAssetProcessor { fn name(&self) -> &'static str { @@ -333,7 +369,8 @@ impl ProcessorTrait for FungibleAssetProcessor { _raw_fungible_asset_balances, (raw_current_unified_fab_v1, raw_current_unified_fab_v2), mut coin_supply, - ) = parse_v2_coin(&transactions).await; + fa_to_coin_mappings, + ) = parse_v2_coin(&transactions, None).await; let postgres_fungible_asset_activities: Vec = raw_fungible_asset_activities @@ -358,6 +395,11 @@ impl ProcessorTrait for FungibleAssetProcessor { .map(CurrentUnifiedFungibleAssetBalance::from_raw) .collect(); + let postgres_fa_to_coin_mappings: Vec = fa_to_coin_mappings + .into_iter() + .map(FungibleAssetToCoinMapping::from_raw) + .collect(); + let processing_duration_in_secs = processing_start.elapsed().as_secs_f64(); let db_insertion_start = std::time::Instant::now(); @@ -386,6 +428,7 @@ impl ProcessorTrait for FungibleAssetProcessor { &postgres_current_unified_fab_v2, ), &coin_supply, + &postgres_fa_to_coin_mappings, &self.per_table_chunk_sizes, ) .await; @@ -418,10 +461,63 @@ impl ProcessorTrait for FungibleAssetProcessor { } } +/// Gets coin to fungible asset mappings from transactions by looking at CoinInfo +/// This is very similar code to part of parse_v2_coin +pub async fn get_fa_to_coin_mapping(transactions: &[Transaction]) -> FungibleAssetToCoinMappings { + // First collect all metadata from transactions + let data: Vec<_> = transactions + .par_iter() + .map(|txn| { + let mut kv_mapping: FungibleAssetToCoinMappings = AHashMap::new(); + + let txn_version = txn.version as i64; + let transaction_info = txn.info.as_ref().expect("Transaction info doesn't exist!"); + for (index, wsc) in transaction_info.changes.iter().enumerate() { + if let Change::WriteResource(wr) = wsc.change.as_ref().unwrap() { + if let Some(fa_metadata) = + RawFungibleAssetMetadataModel::get_v1_from_write_resource( + wr, + index as i64, + txn_version, + NaiveDateTime::default(), // placeholder + ) + .unwrap_or_else(|e| { + tracing::error!( + transaction_version = txn_version, + index = index, + error = ?e, + "[Parser] error parsing fungible metadata v1"); + panic!("[Parser] error parsing fungible metadata v1"); + }) + { + let fa_to_coin_mapping = + RawFungibleAssetToCoinMapping::from_raw_fungible_asset_metadata( + &fa_metadata, + ); + kv_mapping.insert( + fa_to_coin_mapping.fungible_asset_metadata_address.clone(), + fa_to_coin_mapping.coin_type.clone(), + ); + } + } + } + kv_mapping + }) + .collect(); + let mut kv_mapping: FungibleAssetToCoinMappings = AHashMap::new(); + for mapping in data { + kv_mapping.extend(mapping); + } + kv_mapping +} + /// TODO: After the migration is complete, we can move this to common models folder /// V2 coin is called fungible assets and this flow includes all data from V1 in coin_processor pub async fn parse_v2_coin( transactions: &[Transaction], + // This mapping is only applied to SDK processor. The old processor will use the hardcoded mapping + // METADATA_TO_COIN_TYPE_MAPPING + persisted_fa_to_coin_mapping: Option<&FungibleAssetToCoinMappings>, ) -> ( Vec, Vec, @@ -431,11 +527,13 @@ pub async fn parse_v2_coin( Vec, ), Vec, + Vec, ) { let mut fungible_asset_activities: Vec = vec![]; let mut fungible_asset_balances: Vec = vec![]; let mut all_coin_supply: Vec = vec![]; let mut fungible_asset_metadata: FungibleAssetMetadataMapping = AHashMap::new(); + let mut fa_to_coin_mappings: FungibleAssetToCoinMappingsForDB = AHashMap::new(); let data: Vec<_> = transactions .par_iter() @@ -444,6 +542,7 @@ pub async fn parse_v2_coin( let mut fungible_asset_metadata = AHashMap::new(); let mut fungible_asset_balances = vec![]; let mut all_coin_supply = vec![]; + let mut fa_to_coin_mappings: FungibleAssetToCoinMappingsForDB = AHashMap::new(); // Get Metadata for fungible assets by object address let mut fungible_asset_object_helper: ObjectAggregatedDataMapping = AHashMap::new(); @@ -458,7 +557,13 @@ pub async fn parse_v2_coin( PROCESSOR_UNKNOWN_TYPE_COUNT .with_label_values(&["FungibleAssetProcessor"]) .inc(); - return (vec![], AHashMap::new(), vec![], vec![]); + return ( + fungible_asset_activities, + fungible_asset_metadata, + fungible_asset_balances, + all_coin_supply, + fa_to_coin_mappings, + ); } let txn_data = txn.txn_data.as_ref().unwrap(); let transaction_info = txn.info.as_ref().expect("Transaction info doesn't exist!"); @@ -666,8 +771,13 @@ pub async fn parse_v2_coin( panic!("[Parser] error parsing fungible metadata v1"); }) { - fungible_asset_metadata - .insert(fa_metadata.asset_type.clone(), fa_metadata); + let asset_type = fa_metadata.asset_type.clone(); + fungible_asset_metadata.insert(asset_type.clone(), fa_metadata.clone()); + let fa_to_coin_mapping = + RawFungibleAssetToCoinMapping::from_raw_fungible_asset_metadata( + &fa_metadata, + ); + fa_to_coin_mappings.insert(asset_type, fa_to_coin_mapping); } if let Some(fa_metadata) = RawFungibleAssetMetadataModel::get_v2_from_write_resource( @@ -726,21 +836,24 @@ pub async fn parse_v2_coin( fungible_asset_metadata, fungible_asset_balances, all_coin_supply, + fa_to_coin_mappings, ) }) .collect(); - for (faa, fam, fab, acs) in data { + for (faa, fam, fab, acs, ctfm) in data { fungible_asset_activities.extend(faa); fungible_asset_balances.extend(fab); all_coin_supply.extend(acs); fungible_asset_metadata.extend(fam); + fa_to_coin_mappings.extend(ctfm); } // Now we need to convert fab into current_unified_fungible_asset_balances v1 and v2 let (current_unified_fab_v1, current_unified_fab_v2) = RawCurrentUnifiedFungibleAssetBalance::from_fungible_asset_balances( &fungible_asset_balances, + persisted_fa_to_coin_mapping, ); // Boilerplate after this @@ -754,17 +867,21 @@ pub async fn parse_v2_coin( let mut current_unified_fab_v2 = current_unified_fab_v2 .into_values() .collect::>(); + let mut fa_to_coin_mapping = fa_to_coin_mappings + .into_values() + .collect::>(); // Sort by PK fungible_asset_metadata.sort_by(|a, b| a.asset_type.cmp(&b.asset_type)); current_unified_fab_v1.sort_by(|a, b| a.storage_id.cmp(&b.storage_id)); current_unified_fab_v2.sort_by(|a, b| a.storage_id.cmp(&b.storage_id)); - + fa_to_coin_mapping.sort_by(|a, b| a.coin_type.cmp(&b.coin_type)); ( fungible_asset_activities, fungible_asset_metadata, fungible_asset_balances, (current_unified_fab_v1, current_unified_fab_v2), all_coin_supply, + fa_to_coin_mapping, ) } diff --git a/rust/processor/src/utils/util.rs b/rust/processor/src/utils/util.rs index dd979bf92..2572b5b07 100644 --- a/rust/processor/src/utils/util.rs +++ b/rust/processor/src/utils/util.rs @@ -469,17 +469,11 @@ pub fn convert_bcs_hex_new(typ: u8, value: String) -> Option { /// Convert the json serialized PropertyMap's inner BCS fields to their original value in string format pub fn convert_bcs_propertymap(s: Value) -> Option { - match PropertyMap::from_bcs_encode_str(s) { - Some(e) => serde_json::to_value(&e).ok(), - None => None, - } + PropertyMap::from_bcs_encode_str(s).and_then(|e| serde_json::to_value(&e).ok()) } pub fn convert_bcs_token_object_propertymap(s: Value) -> Option { - match TokenObjectPropertyMap::from_bcs_encode_str(s) { - Some(e) => serde_json::to_value(&e).ok(), - None => None, - } + TokenObjectPropertyMap::from_bcs_encode_str(s).and_then(|e| serde_json::to_value(&e).ok()) } /// Convert from hex string to raw byte string diff --git a/rust/sdk-processor/src/parquet_processors/parquet_fungible_asset_processor.rs b/rust/sdk-processor/src/parquet_processors/parquet_fungible_asset_processor.rs index 276bd3c82..211e76bb0 100644 --- a/rust/sdk-processor/src/parquet_processors/parquet_fungible_asset_processor.rs +++ b/rust/sdk-processor/src/parquet_processors/parquet_fungible_asset_processor.rs @@ -118,9 +118,10 @@ impl ProcessorTrait for ParquetFungibleAssetProcessor { .await?; let backfill_table = set_backfill_table_flag(parquet_processor_config.backfill_table); - let parquet_fa_extractor = ParquetFungibleAssetExtractor { - opt_in_tables: backfill_table, - }; + let mut parquet_fa_extractor = ParquetFungibleAssetExtractor::new(backfill_table); + parquet_fa_extractor + .bootstrap_fa_to_coin_mapping(self.db_pool.clone()) + .await?; let gcs_client = initialize_gcs_client(parquet_db_config.google_application_credentials.clone()).await; diff --git a/rust/sdk-processor/src/processors/fungible_asset_processor.rs b/rust/sdk-processor/src/processors/fungible_asset_processor.rs index d2c66db48..19a2ca9a2 100644 --- a/rust/sdk-processor/src/processors/fungible_asset_processor.rs +++ b/rust/sdk-processor/src/processors/fungible_asset_processor.rs @@ -101,7 +101,11 @@ impl ProcessorTrait for FungibleAssetProcessor { ..self.config.transaction_stream_config.clone() }) .await?; - let fa_extractor = FungibleAssetExtractor {}; + + let mut fa_extractor = FungibleAssetExtractor::new(); + fa_extractor + .bootstrap_fa_to_coin_mapping(self.db_pool.clone()) + .await?; let fa_storer = FungibleAssetStorer::new( self.db_pool.clone(), processor_config.clone(), diff --git a/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_extractor.rs b/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_extractor.rs index 6d3129294..9b2303c6d 100644 --- a/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_extractor.rs +++ b/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_extractor.rs @@ -1,3 +1,6 @@ +use crate::utils::database::ArcDbPool; +use ahash::AHashMap; +use anyhow::Result; use aptos_indexer_processor_sdk::{ aptos_protos::transaction::v1::Transaction, traits::{async_step::AsyncRunType, AsyncStep, NamedStep, Processable}, @@ -12,6 +15,10 @@ use processor::{ raw_v2_fungible_asset_balances::{ CurrentUnifiedFungibleAssetBalanceConvertible, FungibleAssetBalanceConvertible, }, + raw_v2_fungible_asset_to_coin_mappings::{ + FungibleAssetToCoinMappingConvertible, FungibleAssetToCoinMappings, + RawFungibleAssetToCoinMapping, + }, raw_v2_fungible_metadata::FungibleAssetMetadataConvertible, }, postgres::models::{ @@ -21,17 +28,49 @@ use processor::{ v2_fungible_asset_balances::{ CurrentUnifiedFungibleAssetBalance, FungibleAssetBalance, }, + v2_fungible_asset_to_coin_mappings::FungibleAssetToCoinMapping, v2_fungible_metadata::FungibleAssetMetadataModel, }, }, }, - processors::fungible_asset_processor::parse_v2_coin, + processors::fungible_asset_processor::{get_fa_to_coin_mapping, parse_v2_coin}, }; /// Extracts fungible asset events, metadata, balances, and v1 supply from transactions pub struct FungibleAssetExtractor where - Self: Sized + Send + 'static, {} + Self: Sized + Send + 'static, +{ + pub fa_to_coin_mapping: FungibleAssetToCoinMappings, +} + +impl FungibleAssetExtractor { + pub fn new() -> Self { + Self { + fa_to_coin_mapping: AHashMap::new(), + } + } + + pub async fn bootstrap_fa_to_coin_mapping(&mut self, db_pool: ArcDbPool) -> Result<()> { + tracing::info!("Started bootstrapping fungible asset to coin mapping"); + let start = std::time::Instant::now(); + let mut conn = db_pool.get().await?; + let mapping = RawFungibleAssetToCoinMapping::get_all_mappings(&mut conn).await; + self.fa_to_coin_mapping = mapping; + tracing::info!( + item_count = self.fa_to_coin_mapping.len(), + duration_ms = start.elapsed().as_millis(), + "Finished bootstrapping fungible asset to coin mapping" + ); + Ok(()) + } +} + +impl Default for FungibleAssetExtractor { + fn default() -> Self { + Self::new() + } +} #[async_trait] impl Processable for FungibleAssetExtractor { @@ -45,6 +84,7 @@ impl Processable for FungibleAssetExtractor { Vec, ), Vec, + Vec, ); type RunType = AsyncRunType; @@ -62,17 +102,22 @@ impl Processable for FungibleAssetExtractor { Vec, ), Vec, + Vec, )>, >, ProcessorError, > { + let new_fa_to_coin_mapping = get_fa_to_coin_mapping(&transactions.data).await; + // Merge the mappings + self.fa_to_coin_mapping.extend(new_fa_to_coin_mapping); let ( raw_fungible_asset_activities, raw_fungible_asset_metadata, raw_fungible_asset_balances, (raw_current_unified_fab_v1, raw_current_unified_fab_v2), coin_supply, - ) = parse_v2_coin(&transactions.data).await; + fa_to_coin_mappings, + ) = parse_v2_coin(&transactions.data, Some(&self.fa_to_coin_mapping)).await; let postgres_fungible_asset_activities: Vec = raw_fungible_asset_activities @@ -102,6 +147,10 @@ impl Processable for FungibleAssetExtractor { .into_iter() .map(CurrentUnifiedFungibleAssetBalance::from_raw) .collect(); + let postgres_fa_to_coin_mappings: Vec = fa_to_coin_mappings + .into_iter() + .map(FungibleAssetToCoinMapping::from_raw) + .collect(); Ok(Some(TransactionContext { data: ( @@ -113,6 +162,7 @@ impl Processable for FungibleAssetExtractor { postgres_current_unified_fab_v2, ), coin_supply, + postgres_fa_to_coin_mappings, ), metadata: transactions.metadata, })) diff --git a/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_storer.rs b/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_storer.rs index 19bffd830..1e3879eb6 100644 --- a/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_storer.rs +++ b/rust/sdk-processor/src/steps/fungible_asset_processor/fungible_asset_storer.rs @@ -18,6 +18,7 @@ use processor::{ v2_fungible_asset_balances::{ CurrentUnifiedFungibleAssetBalance, FungibleAssetBalance, }, + v2_fungible_asset_to_coin_mappings::FungibleAssetToCoinMapping, v2_fungible_metadata::FungibleAssetMetadataModel, }, }, @@ -25,6 +26,7 @@ use processor::{ insert_coin_supply_query, insert_current_unified_fungible_asset_balances_v1_query, insert_current_unified_fungible_asset_balances_v2_query, insert_fungible_asset_activities_query, insert_fungible_asset_metadata_query, + insert_fungible_asset_to_coin_mappings_query, }, utils::table_flags::TableFlags, }; @@ -63,6 +65,7 @@ impl Processable for FungibleAssetStorer { Vec, ), Vec, + Vec, ); type Output = (); type RunType = AsyncRunType; @@ -78,14 +81,16 @@ impl Processable for FungibleAssetStorer { Vec, ), Vec, + Vec, )>, ) -> Result>, ProcessorError> { let ( - fungible_asset_activities, - fungible_asset_metadata, + mut fungible_asset_activities, + mut fungible_asset_metadata, mut fungible_asset_balances, (mut current_unified_fab_v1, mut current_unified_fab_v2), mut coin_supply, + fa_to_coin_mappings, ) = input.data; let per_table_chunk_sizes: AHashMap = @@ -108,6 +113,20 @@ impl Processable for FungibleAssetStorer { coin_supply.clear(); } + if self + .deprecated_tables + .contains(TableFlags::FUNGIBLE_ASSET_ACTIVITIES) + { + fungible_asset_activities.clear(); + } + + if self + .deprecated_tables + .contains(TableFlags::FUNGIBLE_ASSET_METADATA) + { + fungible_asset_metadata.clear(); + } + let faa = execute_in_chunks( self.conn_pool.clone(), insert_fungible_asset_activities_query, @@ -150,9 +169,18 @@ impl Processable for FungibleAssetStorer { &coin_supply, get_config_table_chunk_size::("coin_supply", &per_table_chunk_sizes), ); - let (faa_res, fam_res, cufab1_res, cufab2_res, cs_res) = - tokio::join!(faa, fam, cufab_v1, cufab_v2, cs); - for res in [faa_res, fam_res, cufab1_res, cufab2_res, cs_res] { + let fatcm = execute_in_chunks( + self.conn_pool.clone(), + insert_fungible_asset_to_coin_mappings_query, + &fa_to_coin_mappings, + get_config_table_chunk_size::( + "fungible_asset_to_coin_mappings", + &per_table_chunk_sizes, + ), + ); + let (faa_res, fam_res, cufab1_res, cufab2_res, cs_res, fatcm_res) = + tokio::join!(faa, fam, cufab_v1, cufab_v2, cs, fatcm); + for res in [faa_res, fam_res, cufab1_res, cufab2_res, cs_res, fatcm_res] { match res { Ok(_) => {}, Err(e) => { diff --git a/rust/sdk-processor/src/steps/parquet_fungible_asset_processor/parquet_fa_extractor.rs b/rust/sdk-processor/src/steps/parquet_fungible_asset_processor/parquet_fa_extractor.rs index 79211bbf9..fba2f0025 100644 --- a/rust/sdk-processor/src/steps/parquet_fungible_asset_processor/parquet_fa_extractor.rs +++ b/rust/sdk-processor/src/steps/parquet_fungible_asset_processor/parquet_fa_extractor.rs @@ -2,6 +2,7 @@ use crate::{ parquet_processors::{ParquetTypeEnum, ParquetTypeStructs}, utils::parquet_extractor_helper::add_to_map_if_opted_in_for_backfill, }; +use ahash::AHashMap; use aptos_indexer_processor_sdk::{ aptos_protos::transaction::v1::Transaction, traits::{async_step::AsyncRunType, AsyncStep, NamedStep, Processable}, @@ -14,6 +15,9 @@ use processor::{ common::models::fungible_asset_models::{ raw_v2_fungible_asset_activities::FungibleAssetActivityConvertible, raw_v2_fungible_asset_balances::FungibleAssetBalanceConvertible, + raw_v2_fungible_asset_to_coin_mappings::{ + FungibleAssetToCoinMappings, RawFungibleAssetToCoinMapping, + }, raw_v2_fungible_metadata::FungibleAssetMetadataConvertible, }, parquet::models::fungible_asset_models::{ @@ -22,8 +26,8 @@ use processor::{ parquet_v2_fungible_metadata::FungibleAssetMetadataModel, }, }, - processors::fungible_asset_processor::parse_v2_coin, - utils::table_flags::TableFlags, + processors::fungible_asset_processor::{get_fa_to_coin_mapping, parse_v2_coin}, + utils::{database::ArcDbPool, table_flags::TableFlags}, }; use std::collections::HashMap; use tracing::debug; @@ -34,6 +38,23 @@ where Self: Processable + Send + Sized + 'static, { pub opt_in_tables: TableFlags, + pub fa_to_coin_mapping: FungibleAssetToCoinMappings, +} + +impl ParquetFungibleAssetExtractor { + pub fn new(opt_in_tables: TableFlags) -> Self { + Self { + opt_in_tables, + fa_to_coin_mapping: AHashMap::new(), + } + } + + pub async fn bootstrap_fa_to_coin_mapping(&mut self, db_pool: ArcDbPool) -> anyhow::Result<()> { + let mut conn = db_pool.get().await?; + let mapping = RawFungibleAssetToCoinMapping::get_all_mappings(&mut conn).await; + self.fa_to_coin_mapping = mapping; + Ok(()) + } } type ParquetTypeMap = HashMap; @@ -48,13 +69,19 @@ impl Processable for ParquetFungibleAssetExtractor { &mut self, transactions: TransactionContext, ) -> anyhow::Result>, ProcessorError> { + // get the new fa_to_coin_mapping from the transactions + let new_fa_to_coin_mapping = get_fa_to_coin_mapping(&transactions.data).await; + // Merge the mappings + self.fa_to_coin_mapping.extend(new_fa_to_coin_mapping); + let ( raw_fungible_asset_activities, raw_fungible_asset_metadata, raw_fungible_asset_balances, - (_raw_current_unified_fab_v1, _raw_current_unified_fab_v2), + _, _raw_coin_supply, - ) = parse_v2_coin(&transactions.data).await; + _raw_fa_to_coin_mappings, + ) = parse_v2_coin(&transactions.data, Some(&self.fa_to_coin_mapping)).await; let parquet_fungible_asset_activities: Vec = raw_fungible_asset_activities From e943d0fb8e6b7f196a5a6eb666f7d7ba8c55174b Mon Sep 17 00:00:00 2001 From: Yuun Lim <38443641+yuunlimm@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:23:59 -0800 Subject: [PATCH 4/4] support multi txn test (#703) ### Description This is to help testing a sequential multi txn tests particularly for FA processor at the moment. this is done by refactoring some of codes we already have. Which means we still support a single batch of multi txns at the level of Testing Framework. We will be working on some improvements to testing framework to: 1. Support better file generation (right now it creates two folders for one test case in the same level) 2. Support Sequential Multi txn Testing at the level of testing framework. Test cases That Testing Framework supports: ### A: Sequential Multi Txn testing you have multiple txns as an input, processor processing order in two or more processing. Example: ``` Input: [txn1, txn2] txn1 from Grpc, processor processes txn1 txn2 from Grpc, processor processes txn2 ``` ### B: Single Batch of multi txn testing Example: ``` input: [txn1, txn2] 1. txn1, txn2, process 2. txn3, process ``` ### C. Combo of A and B: Example: ``` input: [txn1, txn2, txn3, ... , n] [txn1, txn2, txn3, ... , n] and processor processes ``` --- rust/Cargo.lock | 6 +- rust/Cargo.toml | 2 +- .../asset_type_null/coin_supply.json | 18 ++ .../current_fungible_asset_balances.json | 192 ++++++++++++++ .../fungible_asset_activities.json | 236 ++++++++++++++++++ .../fungible_asset_balances.json | 1 + .../fungible_asset_metadata.json | 1 + .../fa_migration_2_batch/coin_supply.json | 18 ++ .../current_fungible_asset_balances.json | 78 ++++++ .../fungible_asset_activities.json | 74 ++++++ .../fungible_asset_balances.json | 1 + .../fungible_asset_metadata.json | 36 +++ .../fungible_asset_processor.rs | 34 ++- .../fungible_asset_processor_tests.rs | 164 ++++++++---- 14 files changed, 790 insertions(+), 71 deletions(-) create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/coin_supply.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/current_fungible_asset_balances.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_activities.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_balances.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_metadata.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/coin_supply.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/current_fungible_asset_balances.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_activities.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_balances.json create mode 100644 rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_metadata.json diff --git a/rust/Cargo.lock b/rust/Cargo.lock index f5ea5c716..ed2657dbb 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "aptos-indexer-test-transactions" version = "1.0.0" -source = "git+https://github.com/aptos-labs/aptos-core.git?rev=b91d6bfe955e0df12927ae06f44c9763f5f016d3#b91d6bfe955e0df12927ae06f44c9763f5f016d3" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=6f64aea8e5aba9207ffce13060b8bdd031a8de31#6f64aea8e5aba9207ffce13060b8bdd031a8de31" dependencies = [ - "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=b91d6bfe955e0df12927ae06f44c9763f5f016d3)", + "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=6f64aea8e5aba9207ffce13060b8bdd031a8de31)", "serde_json", ] @@ -309,7 +309,7 @@ dependencies = [ [[package]] name = "aptos-protos" version = "1.3.1" -source = "git+https://github.com/aptos-labs/aptos-core.git?rev=b91d6bfe955e0df12927ae06f44c9763f5f016d3#b91d6bfe955e0df12927ae06f44c9763f5f016d3" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=6f64aea8e5aba9207ffce13060b8bdd031a8de31#6f64aea8e5aba9207ffce13060b8bdd031a8de31" dependencies = [ "pbjson", "prost 0.13.4", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b5d51df30..35572a9e0 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,7 +31,7 @@ anyhow = "1.0.86" aptos-indexer-processor-sdk = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "b79ed8b5864b2a12a1f9c5fd01579462e029b2ae" } aptos-indexer-processor-sdk-server-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "b79ed8b5864b2a12a1f9c5fd01579462e029b2ae" } aptos-protos = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "1d8460a995503574ec4e9699d3442d0150d7f3b9" } -aptos-indexer-test-transactions = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "b91d6bfe955e0df12927ae06f44c9763f5f016d3" } +aptos-indexer-test-transactions = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "6f64aea8e5aba9207ffce13060b8bdd031a8de31" } aptos-indexer-testing-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "b79ed8b5864b2a12a1f9c5fd01579462e029b2ae" } async-trait = "0.1.53" backtrace = "0.3.58" diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/coin_supply.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/coin_supply.json new file mode 100644 index 000000000..c209075f3 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/coin_supply.json @@ -0,0 +1,18 @@ +[ + { + "transaction_version": 2308282694, + "coin_type_hash": "91ceb1308a98389691e05158b07ed5f079ab78461a6bb8d5a4054b1bb5cb8bb6", + "coin_type": "0x1::aptos_coin::AptosCoin", + "supply": "113830046844937808", + "transaction_timestamp": "2025-02-01T17:05:00", + "transaction_epoch": 10169 + }, + { + "transaction_version": 2308283617, + "coin_type_hash": "91ceb1308a98389691e05158b07ed5f079ab78461a6bb8d5a4054b1bb5cb8bb6", + "coin_type": "0x1::aptos_coin::AptosCoin", + "supply": "113830046828042038", + "transaction_timestamp": "2025-02-01T17:05:15", + "transaction_epoch": 10169 + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/current_fungible_asset_balances.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/current_fungible_asset_balances.json new file mode 100644 index 000000000..e78ec86b7 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/current_fungible_asset_balances.json @@ -0,0 +1,192 @@ +[ + { + "storage_id": "0x2829f224066c8b5a925d277e88c9ff7f72b44c40b01a0995f1e2773ebaeca412", + "owner_address": "0x3656f551dbdddb61a5e0956b8f64fd9f9d20c48c9f8f2a07551072fd15a3cddb", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "115070470560", + "amount_v2": null, + "amount": "115070470560", + "last_transaction_version_v1": 2308283617, + "last_transaction_version_v2": null, + "last_transaction_version": 2308283617, + "last_transaction_timestamp_v1": "2025-02-01T17:05:15", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:15", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0x3adab3cb7723d75180dae65c7e75802cd9adec7eb387f50bec250d33d3313d36", + "owner_address": "0xb7cb159db88215cd1670edfe4e30df58177571610983fc06681f4c9773810b3e", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "0", + "amount_v2": null, + "amount": "0", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0x48e8342e9d33174172b0ef9f0a8333018f174a41de02d90bce68f46994e15917", + "owner_address": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "912327475722", + "amount_v2": null, + "amount": "912327475722", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0x591c18b4621f2e4703febc771289065882f612e059b9734c49ad56101dd43b7f", + "owner_address": "0xc05e013f9f81d699e5a991bd134ab8a9af4ec609714f1fd8bfce4fc38419c890", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "114558714630", + "amount_v2": null, + "amount": "114558714630", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0x6633f87950212c7ae22c5eb4410589fe265ffc046ad64b32d30091b6278ef1d0", + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "asset_type_v2": "0x3be35c307c15290ec13c2294d40129ddd27e5b96050ec8d43b8aa30255957088", + "asset_type_v1": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "121586567739", + "amount_v2": "100000000", + "amount": "121686567739", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": 2308283617, + "last_transaction_version": 2308283617, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": "2025-02-01T17:05:15", + "last_transaction_timestamp": "2025-02-01T17:05:15", + "token_standard": "v1", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin" + }, + { + "storage_id": "0x6c30c7bce63ed412137a0d61c8c6e7e78c3cd0b07bc7a297b8920cef8d9eda89", + "owner_address": "0x97670e81558c0389314f57e9b14c18db37ec7a823747c1bac176553d07446653", + "asset_type_v2": "0x3be35c307c15290ec13c2294d40129ddd27e5b96050ec8d43b8aa30255957088", + "asset_type_v1": null, + "is_primary": false, + "is_frozen": false, + "amount_v1": null, + "amount_v2": "84903000000", + "amount": "84903000000", + "last_transaction_version_v1": null, + "last_transaction_version_v2": 2308283617, + "last_transaction_version": 2308283617, + "last_transaction_timestamp_v1": null, + "last_transaction_timestamp_v2": "2025-02-01T17:05:15", + "last_transaction_timestamp": "2025-02-01T17:05:15", + "token_standard": "v2", + "asset_type": "0x3be35c307c15290ec13c2294d40129ddd27e5b96050ec8d43b8aa30255957088" + }, + { + "storage_id": "0xbc52b11f809e5c13236f3ec3296f56259b7bee9898d47efd621354083adb97d9", + "owner_address": "0xd0b17bea776bb87b70b2fb2ca631014f0ca94fc1acde4b8ff1a763f4172aa6c4", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "42543253825", + "amount_v2": null, + "amount": "42543253825", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0xc4fc38001ad5d8c1d85b6908c284deb29cc94faf5219061a3db6443397ba3fbf", + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "183280722", + "amount_v2": null, + "amount": "183280722", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0xc6fa431d283690e248c267e56f8062a49d09e81747b85cc5239d0590c40c6a45", + "owner_address": "0xb7cb159db88215cd1670edfe4e30df58177571610983fc06681f4c9773810b3e", + "asset_type_v2": null, + "asset_type_v1": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "0", + "amount_v2": null, + "amount": "0", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin" + }, + { + "storage_id": "0xea2c6900b9e681af5478e679b2d20bb5f73d30534f8a420601cd019997d2e0f6", + "owner_address": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d", + "asset_type_v2": null, + "asset_type_v1": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "187091465210174", + "amount_v2": null, + "amount": "187091465210174", + "last_transaction_version_v1": 2308282694, + "last_transaction_version_v2": null, + "last_transaction_version": 2308282694, + "last_transaction_timestamp_v1": "2025-02-01T17:05:00", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2025-02-01T17:05:00", + "token_standard": "v1", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_activities.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_activities.json new file mode 100644 index 000000000..94e712b16 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_activities.json @@ -0,0 +1,236 @@ +[ + { + "transaction_version": 2308282694, + "event_index": -1, + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "storage_id": "0xc4fc38001ad5d8c1d85b6908c284deb29cc94faf5219061a3db6443397ba3fbf", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "12000", + "type_": "0x1::aptos_coin::GasFeeEvent", + "is_gas_fee": true, + "gas_fee_payer_address": "0xc05e013f9f81d699e5a991bd134ab8a9af4ec609714f1fd8bfce4fc38419c890", + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 0, + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "storage_id": "0xc4fc38001ad5d8c1d85b6908c284deb29cc94faf5219061a3db6443397ba3fbf", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "600000000", + "type_": "0x1::coin::WithdrawEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 1, + "owner_address": "0xb7cb159db88215cd1670edfe4e30df58177571610983fc06681f4c9773810b3e", + "storage_id": "0x3adab3cb7723d75180dae65c7e75802cd9adec7eb387f50bec250d33d3313d36", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "600000000", + "type_": "0x1::coin::DepositEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 4, + "owner_address": "0xb7cb159db88215cd1670edfe4e30df58177571610983fc06681f4c9773810b3e", + "storage_id": "0x3adab3cb7723d75180dae65c7e75802cd9adec7eb387f50bec250d33d3313d36", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "600000000", + "type_": "0x1::coin::WithdrawEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 5, + "owner_address": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d", + "storage_id": "0x48e8342e9d33174172b0ef9f0a8333018f174a41de02d90bce68f46994e15917", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "594000000", + "type_": "0x1::coin::DepositEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 6, + "owner_address": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d", + "storage_id": "0xea2c6900b9e681af5478e679b2d20bb5f73d30534f8a420601cd019997d2e0f6", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_frozen": null, + "amount": "121586320899", + "type_": "0x1::coin::WithdrawEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 7, + "owner_address": "0xb7cb159db88215cd1670edfe4e30df58177571610983fc06681f4c9773810b3e", + "storage_id": "0xc6fa431d283690e248c267e56f8062a49d09e81747b85cc5239d0590c40c6a45", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_frozen": null, + "amount": "121586320899", + "type_": "0x1::coin::DepositEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 8, + "owner_address": "0xd0b17bea776bb87b70b2fb2ca631014f0ca94fc1acde4b8ff1a763f4172aa6c4", + "storage_id": "0xbc52b11f809e5c13236f3ec3296f56259b7bee9898d47efd621354083adb97d9", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "6000000", + "type_": "0x1::coin::DepositEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 11, + "owner_address": "0xb7cb159db88215cd1670edfe4e30df58177571610983fc06681f4c9773810b3e", + "storage_id": "0xc6fa431d283690e248c267e56f8062a49d09e81747b85cc5239d0590c40c6a45", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_frozen": null, + "amount": "121586320899", + "type_": "0x1::coin::WithdrawEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308282694, + "event_index": 14, + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "storage_id": "0x6633f87950212c7ae22c5eb4410589fe265ffc046ad64b32d30091b6278ef1d0", + "asset_type": "0x17740e230cb5ac3f6eb16135fa1ce02baf9a07f2acfa884b33b0fb2f1bc2b91d::coin_factory::Emojicoin", + "is_frozen": null, + "amount": "121586320899", + "type_": "0x1::coin::DepositEvent", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x1c3206329806286fd2223647c9f9b130e66baeb6d7224a18c1f642ffe48f3b4c::panora_swap::router_entry", + "block_height": 284832501, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:00", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308283617, + "event_index": -1, + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "storage_id": "0xc4fc38001ad5d8c1d85b6908c284deb29cc94faf5219061a3db6443397ba3fbf", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "58400", + "type_": "0x1::aptos_coin::GasFeeEvent", + "is_gas_fee": true, + "gas_fee_payer_address": "0x3656f551dbdddb61a5e0956b8f64fd9f9d20c48c9f8f2a07551072fd15a3cddb", + "is_transaction_success": true, + "entry_function_id_str": "0x67b0be11ef916e46d0d045ddde44285cde84df2e314682b32e22a1d32f4bc0a0::PanoraClaims::claim", + "block_height": 284832579, + "token_standard": "v1", + "transaction_timestamp": "2025-02-01T17:05:15", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308283617, + "event_index": 0, + "owner_address": "0x97670e81558c0389314f57e9b14c18db37ec7a823747c1bac176553d07446653", + "storage_id": "0x6c30c7bce63ed412137a0d61c8c6e7e78c3cd0b07bc7a297b8920cef8d9eda89", + "asset_type": "0x3be35c307c15290ec13c2294d40129ddd27e5b96050ec8d43b8aa30255957088", + "is_frozen": null, + "amount": "100000000", + "type_": "0x1::fungible_asset::Withdraw", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x67b0be11ef916e46d0d045ddde44285cde84df2e314682b32e22a1d32f4bc0a0::PanoraClaims::claim", + "block_height": 284832579, + "token_standard": "v2", + "transaction_timestamp": "2025-02-01T17:05:15", + "storage_refund_amount": "0" + }, + { + "transaction_version": 2308283617, + "event_index": 1, + "owner_address": "0x17ca1ee00a2baaee0a0419bd8ef25489b96ece1906881ae702b317eefd940076", + "storage_id": "0x6633f87950212c7ae22c5eb4410589fe265ffc046ad64b32d30091b6278ef1d0", + "asset_type": "0x3be35c307c15290ec13c2294d40129ddd27e5b96050ec8d43b8aa30255957088", + "is_frozen": null, + "amount": "100000000", + "type_": "0x1::fungible_asset::Deposit", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x67b0be11ef916e46d0d045ddde44285cde84df2e314682b32e22a1d32f4bc0a0::PanoraClaims::claim", + "block_height": 284832579, + "token_standard": "v2", + "transaction_timestamp": "2025-02-01T17:05:15", + "storage_refund_amount": "0" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_balances.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_balances.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_balances.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_metadata.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_metadata.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/asset_type_null/fungible_asset_metadata.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/coin_supply.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/coin_supply.json new file mode 100644 index 000000000..9dbbad049 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/coin_supply.json @@ -0,0 +1,18 @@ +[ + { + "transaction_version": 1680592683, + "coin_type_hash": "91ceb1308a98389691e05158b07ed5f079ab78461a6bb8d5a4054b1bb5cb8bb6", + "coin_type": "0x1::aptos_coin::AptosCoin", + "supply": "111442820397688086", + "transaction_timestamp": "2024-09-05T11:52:16", + "transaction_epoch": 8371 + }, + { + "transaction_version": 1957950162, + "coin_type_hash": "91ceb1308a98389691e05158b07ed5f079ab78461a6bb8d5a4054b1bb5cb8bb6", + "coin_type": "0x1::aptos_coin::AptosCoin", + "supply": "112743621173497128", + "transaction_timestamp": "2024-11-23T22:32:30", + "transaction_epoch": 9331 + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/current_fungible_asset_balances.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/current_fungible_asset_balances.json new file mode 100644 index 000000000..dae8cce2c --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/current_fungible_asset_balances.json @@ -0,0 +1,78 @@ +[ + { + "storage_id": "0x1734f2619bb775e59d04469663b9c05614b5ada6afbf2f750095f43cb8d41e40", + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "115245132", + "amount_v2": null, + "amount": "115245132", + "last_transaction_version_v1": 1957950162, + "last_transaction_version_v2": null, + "last_transaction_version": 1957950162, + "last_transaction_timestamp_v1": "2024-11-23T22:32:30", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2024-11-23T22:32:30", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + }, + { + "storage_id": "0xa1ddab83c8209aabffc66264730ba95bcc7d7a7966192c00d800ab734729622c", + "owner_address": "0x7ae7cc51c4fab67181a969f53a5d01a292dad0baf259c92c4e1a13f056768e1c", + "asset_type_v2": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "asset_type_v1": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL", + "is_primary": true, + "is_frozen": false, + "amount_v1": null, + "amount_v2": "431101567356", + "amount": "431101567356", + "last_transaction_version_v1": null, + "last_transaction_version_v2": 1957950162, + "last_transaction_version": 1957950162, + "last_transaction_timestamp_v1": null, + "last_transaction_timestamp_v2": "2024-11-23T22:32:30", + "last_transaction_timestamp": "2024-11-23T22:32:30", + "token_standard": "v1", + "asset_type": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL" + }, + { + "storage_id": "0xd29b17433643769f27d89851553ec2ab5e4204b66eceecb6eec774668117c3e0", + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "asset_type_v2": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "asset_type_v1": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL", + "is_primary": true, + "is_frozen": false, + "amount_v1": null, + "amount_v2": "3643926", + "amount": "3643926", + "last_transaction_version_v1": null, + "last_transaction_version_v2": 1957950162, + "last_transaction_version": 1957950162, + "last_transaction_timestamp_v1": null, + "last_transaction_timestamp_v2": "2024-11-23T22:32:30", + "last_transaction_timestamp": "2024-11-23T22:32:30", + "token_standard": "v1", + "asset_type": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL" + }, + { + "storage_id": "0xfa4e15c47bc6b38cc404cf08e30e69fd14ff2cc82b136ef3630445a66605653c", + "owner_address": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06", + "asset_type_v2": null, + "asset_type_v1": "0x1::aptos_coin::AptosCoin", + "is_primary": true, + "is_frozen": false, + "amount_v1": "997323833", + "amount_v2": null, + "amount": "997323833", + "last_transaction_version_v1": 1680592683, + "last_transaction_version_v2": null, + "last_transaction_version": 1680592683, + "last_transaction_timestamp_v1": "2024-09-05T11:52:16", + "last_transaction_timestamp_v2": null, + "last_transaction_timestamp": "2024-09-05T11:52:16", + "token_standard": "v1", + "asset_type": "0x1::aptos_coin::AptosCoin" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_activities.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_activities.json new file mode 100644 index 000000000..ecd37babc --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_activities.json @@ -0,0 +1,74 @@ +[ + { + "transaction_version": 1680592683, + "event_index": -1, + "owner_address": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06", + "storage_id": "0xfa4e15c47bc6b38cc404cf08e30e69fd14ff2cc82b136ef3630445a66605653c", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "205700", + "type_": "0x1::aptos_coin::GasFeeEvent", + "is_gas_fee": true, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_mkl_token::initialize_module", + "block_height": 222731447, + "token_standard": "v1", + "transaction_timestamp": "2024-09-05T11:52:16", + "storage_refund_amount": "47280" + }, + { + "transaction_version": 1957950162, + "event_index": -1, + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "storage_id": "0x1734f2619bb775e59d04469663b9c05614b5ada6afbf2f750095f43cb8d41e40", + "asset_type": "0x1::aptos_coin::AptosCoin", + "is_frozen": null, + "amount": "2100", + "type_": "0x1::aptos_coin::GasFeeEvent", + "is_gas_fee": true, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_liquidity_auction::claim_mkl_reward", + "block_height": 256094969, + "token_standard": "v1", + "transaction_timestamp": "2024-11-23T22:32:30", + "storage_refund_amount": "0" + }, + { + "transaction_version": 1957950162, + "event_index": 0, + "owner_address": "0x7ae7cc51c4fab67181a969f53a5d01a292dad0baf259c92c4e1a13f056768e1c", + "storage_id": "0xa1ddab83c8209aabffc66264730ba95bcc7d7a7966192c00d800ab734729622c", + "asset_type": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "is_frozen": null, + "amount": "3643926", + "type_": "0x1::fungible_asset::Withdraw", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_liquidity_auction::claim_mkl_reward", + "block_height": 256094969, + "token_standard": "v2", + "transaction_timestamp": "2024-11-23T22:32:30", + "storage_refund_amount": "0" + }, + { + "transaction_version": 1957950162, + "event_index": 1, + "owner_address": "0x345ced5d7c67360d61a1d59d8cd83cb53095a81aa24a337027391b1c7dd1786f", + "storage_id": "0xd29b17433643769f27d89851553ec2ab5e4204b66eceecb6eec774668117c3e0", + "asset_type": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "is_frozen": null, + "amount": "3643926", + "type_": "0x1::fungible_asset::Deposit", + "is_gas_fee": false, + "gas_fee_payer_address": null, + "is_transaction_success": true, + "entry_function_id_str": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::managed_liquidity_auction::claim_mkl_reward", + "block_height": 256094969, + "token_standard": "v2", + "transaction_timestamp": "2024-11-23T22:32:30", + "storage_refund_amount": "0" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_balances.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_balances.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_balances.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_metadata.json b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_metadata.json new file mode 100644 index 000000000..9f8c23199 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/fungible_asset_processor/fa_migration_2_batch/fungible_asset_metadata.json @@ -0,0 +1,36 @@ +[ + { + "asset_type": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06::mkl_token::MKL", + "creator_address": "0x5ae6789dd2fec1a9ec9cccfb3acaf12e93d432f0a3a42c92fe1a9d490b7bbc06", + "name": "MKL", + "symbol": "MKL", + "decimals": 6, + "icon_uri": null, + "project_uri": null, + "last_transaction_version": 1680592683, + "last_transaction_timestamp": "2024-09-05T11:52:16", + "supply_aggregator_table_handle_v1": null, + "supply_aggregator_table_key_v1": null, + "token_standard": "v1", + "maximum_v2": null, + "is_token_v2": null, + "supply_v2": null + }, + { + "asset_type": "0x878370592f9129e14b76558689a4b570ad22678111df775befbfcbc9fb3d90ab", + "creator_address": "0x000000000000000000000000000000000000000000000000000000000000000a", + "name": "MKL", + "symbol": "MKL", + "decimals": 6, + "icon_uri": "", + "project_uri": "", + "last_transaction_version": 1957950162, + "last_transaction_timestamp": "2024-11-23T22:32:30", + "supply_aggregator_table_handle_v1": null, + "supply_aggregator_table_key_v1": null, + "token_standard": "v2", + "maximum_v2": "340282366920938463463374607431768211455", + "is_token_v2": null, + "supply_v2": "96895109643615" + } +] \ No newline at end of file diff --git a/rust/integration-tests/src/diff_test_helper/fungible_asset_processor.rs b/rust/integration-tests/src/diff_test_helper/fungible_asset_processor.rs index d9aa16575..94cd7cf13 100644 --- a/rust/integration-tests/src/diff_test_helper/fungible_asset_processor.rs +++ b/rust/integration-tests/src/diff_test_helper/fungible_asset_processor.rs @@ -3,11 +3,7 @@ use crate::models::fa_v2_models::{ FungibleAssetMetadataModel, }; use anyhow::Result; -use diesel::{ - pg::PgConnection, - query_dsl::methods::{FilterDsl, ThenOrderDsl}, - ExpressionMethods, RunQueryDsl, -}; +use diesel::{pg::PgConnection, ExpressionMethods, QueryDsl, RunQueryDsl}; use processor::schema::{ coin_supply::dsl as cs_dsl, current_fungible_asset_balances::dsl as cfab_dsl, fungible_asset_activities::dsl as faa_dsl, fungible_asset_balances::dsl as fab_dsl, @@ -19,14 +15,15 @@ use std::collections::HashMap; #[allow(dead_code)] pub fn load_data( conn: &mut PgConnection, - txn_versions: Vec, + _txn_versions: Vec, // TODO: Remove this after updating testing framework, for now it's just a placeholder to make the function signature match the original ) -> Result> { let mut result_map: HashMap = HashMap::new(); let fungible_asset_activities_result = faa_dsl::fungible_asset_activities - .filter(faa_dsl::transaction_version.eq_any(&txn_versions)) - .then_order_by(faa_dsl::transaction_version.asc()) - .then_order_by(faa_dsl::event_index.asc()) + .order_by(( + faa_dsl::transaction_version.asc(), + faa_dsl::event_index.asc(), + )) .load::(conn); let all_fungible_asset_activities = fungible_asset_activities_result?; let fungible_asset_activities_json = @@ -37,8 +34,7 @@ pub fn load_data( ); let fungible_asset_metadata_result = fam_dsl::fungible_asset_metadata - .filter(fam_dsl::last_transaction_version.eq_any(&txn_versions)) - .then_order_by(fam_dsl::last_transaction_version.asc()) + .order_by(fam_dsl::last_transaction_version.asc()) .load::(conn); let all_fungible_asset_metadata = fungible_asset_metadata_result?; let fungible_asset_metadata_json = serde_json::to_string_pretty(&all_fungible_asset_metadata)?; @@ -48,9 +44,10 @@ pub fn load_data( ); let fungible_asset_balances_result = fab_dsl::fungible_asset_balances - .filter(fab_dsl::transaction_version.eq_any(&txn_versions)) - .then_order_by(fab_dsl::transaction_version.asc()) - .then_order_by(fab_dsl::write_set_change_index.asc()) + .order_by(( + fab_dsl::transaction_version.asc(), + fab_dsl::write_set_change_index.asc(), + )) .load::(conn); let all_fungible_asset_balances = fungible_asset_balances_result?; let fungible_asset_balances_json = serde_json::to_string_pretty(&all_fungible_asset_balances)?; @@ -60,8 +57,10 @@ pub fn load_data( ); let current_fungible_asset_balances_result = cfab_dsl::current_fungible_asset_balances - .filter(cfab_dsl::last_transaction_version.eq_any(&txn_versions)) - .then_order_by(cfab_dsl::storage_id.asc()) + .order_by(( + cfab_dsl::storage_id.asc(), + cfab_dsl::last_transaction_version.asc(), + )) .load::(conn); let all_current_fungible_asset_balances = current_fungible_asset_balances_result?; let current_fungible_asset_balances_json = @@ -72,8 +71,7 @@ pub fn load_data( ); let coin_supply_result = cs_dsl::coin_supply - .filter(cs_dsl::transaction_version.eq_any(&txn_versions)) - .then_order_by(cs_dsl::transaction_version.asc()) + .order_by(cs_dsl::transaction_version.asc()) .load::(conn); let all_coin_supply = coin_supply_result?; let coin_supply_json = serde_json::to_string_pretty(&all_coin_supply)?; diff --git a/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs b/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs index 9ee3e1ee0..f2e69f1b4 100644 --- a/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/fungible_asset_processor_tests.rs @@ -45,7 +45,7 @@ mod sdk_fungible_asset_processor_tests { diff_test_helper::fungible_asset_processor::load_data, sdk_tests::{ fungible_asset_processor_tests::setup_fa_processor_config, run_processor_test, - setup_test_environment, validate_json, DEFAULT_OUTPUT_FOLDER, + validate_json, DEFAULT_OUTPUT_FOLDER, }, }; use aptos_indexer_test_transactions::json_transactions::generated_transactions::{ @@ -54,6 +54,8 @@ mod sdk_fungible_asset_processor_tests { IMPORTED_MAINNET_TXNS_1737056775_COIN_TRANSFER_BURN_EVENT, IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY, IMPORTED_MAINNET_TXNS_2186504987_COIN_STORE_DELETION_NO_EVENT, + IMPORTED_MAINNET_TXNS_2308282694_ASSET_TYPE_V1_NULL, + IMPORTED_MAINNET_TXNS_2308283617_ASSET_TYPE_V1_NULL_2, IMPORTED_MAINNET_TXNS_255894550_STORAGE_REFUND, IMPORTED_MAINNET_TXNS_508365567_FA_V1_EVENTS, IMPORTED_MAINNET_TXNS_550582915_MULTIPLE_TRANSFER_EVENT, @@ -65,13 +67,17 @@ mod sdk_fungible_asset_processor_tests { IMPORTED_TESTNET_TXNS_5979639459_COIN_REGISTER, IMPORTED_TESTNET_TXNS_5992795934_FA_ACTIVITIES, IMPORTED_TESTNET_TXNS_646928741_NO_EVENTS, }; - use aptos_indexer_testing_framework::{cli_parser::get_test_config, database::TestDatabase}; + use aptos_indexer_testing_framework::{ + cli_parser::get_test_config, + database::{PostgresTestDatabase, TestDatabase}, + sdk_test_context::SdkTestContext, + }; use sdk_processor::processors::fungible_asset_processor::FungibleAssetProcessor; // Test case for processing a specific testnet transaction (Validator Transaction) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_validator_txn() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_5523474016_VALIDATOR_TXN], Some("validator_txn_test".to_string()), ) @@ -81,7 +87,7 @@ mod sdk_fungible_asset_processor_tests { // Test case for processing another testnet transaction (Coin Register) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_register_txn() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_5979639459_COIN_REGISTER], Some("coin_register_txn_test".to_string()), ) @@ -90,7 +96,7 @@ mod sdk_fungible_asset_processor_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_fa_activities_txn() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_5992795934_FA_ACTIVITIES], Some("fa_activities_txn_test".to_string()), ) @@ -112,7 +118,7 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_and_fa_transfers() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_MAINNET_TXNS_999929475_COIN_AND_FA_TRANSFERS], Some("coin_and_fa_transfers_test".to_string()), ) @@ -128,7 +134,7 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_v1_events() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_MAINNET_TXNS_508365567_FA_V1_EVENTS], Some("v1_events_test".to_string()), ) @@ -142,7 +148,7 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_v2_frozen_event() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_1200394037_FA_V2_FROZEN_EVENT], Some("v2_frozen_event_test".to_string()), ) @@ -157,7 +163,7 @@ mod sdk_fungible_asset_processor_tests { */ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_concurrent_fa() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_2646510387_CONCURRENT_FA], Some("concurrent_fa_test".to_string()), ) @@ -168,7 +174,7 @@ mod sdk_fungible_asset_processor_tests { /// Validates the handling of updated coin event formats #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_v2_events() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_DEVNET_TXNS_78753811_COIN_TRANSFER_WITH_V2_EVENTS], Some("coin_v2_events".to_string()), ) @@ -177,7 +183,7 @@ mod sdk_fungible_asset_processor_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_store_deletion_no_event() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_MAINNET_TXNS_2186504987_COIN_STORE_DELETION_NO_EVENT], Some("coin_store_deletion_no_event".to_string()), ) @@ -188,7 +194,7 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of burning tokens from secondary stores #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_secondary_store_burnt() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_4462417704_SECONDARY_STORE_BURNT], Some("secondary_store_burnt".to_string()), ) @@ -199,7 +205,7 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of isolated gas events #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_gas_event_when_events_is_empty() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_TESTNET_TXNS_646928741_NO_EVENTS], Some("gas_event_when_events_is_empty".to_string()), ) @@ -210,7 +216,7 @@ mod sdk_fungible_asset_processor_tests { /// Validates handling of burn operations during coin transfers #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_coin_transfer_burn_event() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_MAINNET_TXNS_1737056775_COIN_TRANSFER_BURN_EVENT], Some("coin_transfer_burn_event".to_string()), ) @@ -221,7 +227,7 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of batch transfers #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_multiple_transfer_event() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_MAINNET_TXNS_550582915_MULTIPLE_TRANSFER_EVENT], Some("multiple_transfer_event".to_string()), ) @@ -232,7 +238,7 @@ mod sdk_fungible_asset_processor_tests { /// Validates correct handling of storage refund mechanics #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_storage_refund() { - process_fa_txns( + process_single_batch_txns( &[IMPORTED_MAINNET_TXNS_255894550_STORAGE_REFUND], Some("storage_refund".to_string()), ) @@ -244,7 +250,7 @@ mod sdk_fungible_asset_processor_tests { /// processes an FA version of that coin #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fungible_asset_processor_fa_migration_same_batch() { - process_fa_txns( + process_single_batch_txns( &[ IMPORTED_MAINNET_TXNS_1680592683_FA_MIGRATION_COIN_INFO, IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY, @@ -255,27 +261,67 @@ mod sdk_fungible_asset_processor_tests { } // TODO: I really want to make this work but it doesn't right now - // #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - // async fn test_fungible_asset_processor_fa_migration_different_batch() { - // process_fa_txns( - // &[IMPORTED_MAINNET_TXNS_1680592683_FA_MIGRATION_COIN_INFO], - // Some("fa_migration_2_batch".to_string()), - // ) - // .await; - - // process_fa_txns( - // &[IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY], - // Some("fa_migration_2_batch".to_string()), - // ) - // .await; - // } - - // Helper function to abstract out the transaction processing - async fn process_fa_txns(txns: &[&[u8]], test_case_name: Option) { - let (diff_flag, custom_output_path) = get_test_config(); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_fungible_asset_processor_fa_migration_different_batch() { + sequential_multi_transaction_helper_function( + &[ + &[IMPORTED_MAINNET_TXNS_1680592683_FA_MIGRATION_COIN_INFO], + &[IMPORTED_MAINNET_TXNS_1957950162_FA_MIGRATION_V2_STORE_ONLY], + ], + "fa_migration_2_batch", + ) + .await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_fungible_asset_processor_asset_type_null() { + sequential_multi_transaction_helper_function( + &[&[IMPORTED_MAINNET_TXNS_2308282694_ASSET_TYPE_V1_NULL], &[ + IMPORTED_MAINNET_TXNS_2308283617_ASSET_TYPE_V1_NULL_2, + ]], + "asset_type_null", + ) + .await; + } + + /// Tests processing of two transactions sequentially + /// Validates handling of multiple transactions with shared context + async fn sequential_multi_transaction_helper_function( + txn_batches: &[&[&[u8]]], + output_name: &str, + ) { + let (generate_flag, custom_output_path) = get_test_config(); let output_path = custom_output_path.unwrap_or_else(|| DEFAULT_OUTPUT_FOLDER.to_string()); - let (db, mut test_context) = setup_test_environment(txns).await; + let mut db = PostgresTestDatabase::new(); + db.setup().await.unwrap(); + + for (i, txn_batch) in txn_batches.iter().enumerate() { + let is_last = i == txn_batches.len() - 1; + process_transactions( + &mut db, + txn_batch, + output_name, + is_last && generate_flag, + &output_path, + is_last, + ) + .await; + } + } + + async fn process_transactions( + db: &mut PostgresTestDatabase, + txns: &[&[u8]], + transaction_name: &str, + generate_flag: bool, + output_path: &str, + should_validate: bool, + ) { + let mut test_context = SdkTestContext::new(txns); + if test_context.init_mock_grpc().await.is_err() { + panic!("Failed to initialize mock grpc"); + }; let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = @@ -290,28 +336,48 @@ mod sdk_fungible_asset_processor_tests { fungible_asset_processor, load_data, db_url, - diff_flag, - output_path.clone(), - test_case_name.clone(), + generate_flag, + output_path.to_string(), + Some(transaction_name.to_string()), ) .await { Ok(mut db_value) => { - let _ = validate_json( - &mut db_value, - test_context.get_request_start_version(), - processor_name, - output_path.clone(), - test_case_name, - ); + if should_validate { + let _ = validate_json( + &mut db_value, + test_context.get_request_start_version(), + processor_name, + output_path.to_string(), + Some(transaction_name.to_string()), + ); + } }, Err(e) => { panic!( - "Test failed on transactions {:?} due to processor error: {}", - test_context.get_test_transaction_versions(), - e + "Test failed on {} due to processor error: {}", + transaction_name, e ); }, } } + + // Update the existing process_single_batch_txns function to use the new merged function + async fn process_single_batch_txns(txns: &[&[u8]], test_case_name: Option) { + let (generate_flag, custom_output_path) = get_test_config(); + let output_path = custom_output_path.unwrap_or_else(|| DEFAULT_OUTPUT_FOLDER.to_string()); + + let mut db = PostgresTestDatabase::new(); + db.setup().await.unwrap(); + + process_transactions( + &mut db, + txns, + &test_case_name.unwrap_or_default(), + generate_flag, + &output_path, + true, // Assuming we want to validate for fungible asset transactions + ) + .await; + } }