-
Notifications
You must be signed in to change notification settings - Fork 11.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
## Description Adding pipelines to index all tables used to filter events. They differ from the equivalent schemas in the existing indexer in the following ways: - They only mention the transaction sequence number, and not the event sequent number. To use these tables, we first filter down to the transaction containing the event, and then scan the events in that transaction. - Struct instantiations are stored as a separate name field and then a BCS encoded type tag. This is to reduce their footprint (package IDs weight twice as much when stored as text compared to BCS), and because we only ever filter using an exact match, so we don't need to store the instantiation as text. ## Test plan Ran the indexer locally and spot checked the events.
- Loading branch information
Showing
13 changed files
with
606 additions
and
0 deletions.
There are no files selected for viewing
6 changes: 6 additions & 0 deletions
6
crates/sui-indexer-alt/migrations/2024-10-19-113135_ev_indices/down.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
DROP TABLE IF EXISTS ev_emit_pkg; | ||
DROP TABLE IF EXISTS ev_emit_mod; | ||
DROP TABLE IF EXISTS ev_struct_pkg; | ||
DROP TABLE IF EXISTS ev_struct_mod; | ||
DROP TABLE IF EXISTS ev_struct_name; | ||
DROP TABLE IF EXISTS ev_struct_inst; |
91 changes: 91 additions & 0 deletions
91
crates/sui-indexer-alt/migrations/2024-10-19-113135_ev_indices/up.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
CREATE TABLE IF NOT EXISTS ev_emit_pkg | ||
( | ||
package BYTEA, | ||
tx_sequence_number BIGINT, | ||
sender BYTEA NOT NULL, | ||
PRIMARY KEY(package, tx_sequence_number) | ||
); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_emit_pkg_tx_sequence_number | ||
ON ev_emit_pkg (tx_sequence_number); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_emit_pkg_sender | ||
ON ev_emit_pkg (sender, package, tx_sequence_number); | ||
|
||
CREATE TABLE IF NOT EXISTS ev_emit_mod | ||
( | ||
package BYTEA, | ||
module TEXT, | ||
tx_sequence_number BIGINT, | ||
sender BYTEA NOT NULL, | ||
PRIMARY KEY(package, module, tx_sequence_number) | ||
); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_emit_mod_tx_sequence_number | ||
ON ev_emit_mod (tx_sequence_number); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_emit_mod_sender | ||
ON ev_emit_mod (sender, package, module, tx_sequence_number); | ||
|
||
CREATE TABLE IF NOT EXISTS ev_struct_pkg | ||
( | ||
package BYTEA, | ||
tx_sequence_number BIGINT, | ||
sender BYTEA NOT NULL, | ||
PRIMARY KEY(package, tx_sequence_number) | ||
); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_pkg_tx_sequence_number | ||
ON ev_struct_pkg (tx_sequence_number); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_pkg_sender | ||
ON ev_struct_pkg (sender, package, tx_sequence_number); | ||
|
||
CREATE TABLE IF NOT EXISTS ev_struct_mod | ||
( | ||
package BYTEA, | ||
module TEXT, | ||
tx_sequence_number BIGINT, | ||
sender BYTEA NOT NULL, | ||
PRIMARY KEY(package, module, tx_sequence_number) | ||
); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_mod_tx_sequence_number | ||
ON ev_struct_mod (tx_sequence_number); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_mod_sender | ||
ON ev_struct_mod (sender, package, module, tx_sequence_number); | ||
|
||
CREATE TABLE IF NOT EXISTS ev_struct_name | ||
( | ||
package BYTEA, | ||
module TEXT, | ||
name TEXT, | ||
tx_sequence_number BIGINT, | ||
sender BYTEA NOT NULL, | ||
PRIMARY KEY(package, module, name, tx_sequence_number) | ||
); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_name_tx_sequence_number | ||
ON ev_struct_name (tx_sequence_number); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_name_sender | ||
ON ev_struct_name (sender, package, module, name, tx_sequence_number); | ||
|
||
CREATE TABLE IF NOT EXISTS ev_struct_inst | ||
( | ||
package BYTEA, | ||
module TEXT, | ||
name TEXT, | ||
-- BCS encoded array of TypeTags for type parameters. | ||
instantiation BYTEA, | ||
tx_sequence_number BIGINT, | ||
sender BYTEA NOT NULL, | ||
PRIMARY KEY(package, module, instantiation, tx_sequence_number) | ||
); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_inst_tx_sequence_number | ||
ON ev_struct_inst (tx_sequence_number); | ||
|
||
CREATE INDEX IF NOT EXISTS ev_struct_inst_sender | ||
ON ev_struct_inst (sender, package, module, instantiation, tx_sequence_number); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use std::{collections::BTreeSet, sync::Arc}; | ||
|
||
use anyhow::Result; | ||
use diesel_async::RunQueryDsl; | ||
use sui_types::full_checkpoint_content::CheckpointData; | ||
|
||
use crate::{db, models::events::StoredEvEmitMod, schema::ev_emit_mod}; | ||
|
||
use super::Handler; | ||
|
||
pub struct EvEmitMod; | ||
|
||
#[async_trait::async_trait] | ||
impl Handler for EvEmitMod { | ||
const NAME: &'static str = "ev_emit_mod"; | ||
|
||
const BATCH_SIZE: usize = 100; | ||
const CHUNK_SIZE: usize = 1000; | ||
const MAX_PENDING_SIZE: usize = 10000; | ||
|
||
type Value = StoredEvEmitMod; | ||
|
||
fn handle(checkpoint: &Arc<CheckpointData>) -> Result<Vec<Self::Value>> { | ||
let CheckpointData { | ||
transactions, | ||
checkpoint_summary, | ||
.. | ||
} = checkpoint.as_ref(); | ||
|
||
let mut values = BTreeSet::new(); | ||
let first_tx = checkpoint_summary.network_total_transactions as usize - transactions.len(); | ||
|
||
for (i, tx) in transactions.iter().enumerate() { | ||
values.extend( | ||
tx.events | ||
.iter() | ||
.flat_map(|evs| &evs.data) | ||
.map(|ev| StoredEvEmitMod { | ||
package: ev.package_id.to_vec(), | ||
module: ev.transaction_module.to_string(), | ||
tx_sequence_number: (first_tx + i) as i64, | ||
sender: ev.sender.to_vec(), | ||
}), | ||
); | ||
} | ||
|
||
Ok(values.into_iter().collect()) | ||
} | ||
|
||
async fn commit(values: &[Self::Value], conn: &mut db::Connection<'_>) -> Result<usize> { | ||
Ok(diesel::insert_into(ev_emit_mod::table) | ||
.values(values) | ||
.on_conflict_do_nothing() | ||
.execute(conn) | ||
.await?) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use std::{collections::BTreeSet, sync::Arc}; | ||
|
||
use anyhow::Result; | ||
use diesel_async::RunQueryDsl; | ||
use sui_types::full_checkpoint_content::CheckpointData; | ||
|
||
use crate::{db, models::events::StoredEvEmitPkg, schema::ev_emit_pkg}; | ||
|
||
use super::Handler; | ||
|
||
pub struct EvEmitPkg; | ||
|
||
#[async_trait::async_trait] | ||
impl Handler for EvEmitPkg { | ||
const NAME: &'static str = "ev_emit_pkg"; | ||
|
||
const BATCH_SIZE: usize = 100; | ||
const CHUNK_SIZE: usize = 1000; | ||
const MAX_PENDING_SIZE: usize = 10000; | ||
|
||
type Value = StoredEvEmitPkg; | ||
|
||
fn handle(checkpoint: &Arc<CheckpointData>) -> Result<Vec<Self::Value>> { | ||
let CheckpointData { | ||
transactions, | ||
checkpoint_summary, | ||
.. | ||
} = checkpoint.as_ref(); | ||
|
||
let mut values = BTreeSet::new(); | ||
let first_tx = checkpoint_summary.network_total_transactions as usize - transactions.len(); | ||
|
||
for (i, tx) in transactions.iter().enumerate() { | ||
values.extend( | ||
tx.events | ||
.iter() | ||
.flat_map(|evs| &evs.data) | ||
.map(|ev| StoredEvEmitPkg { | ||
package: ev.package_id.to_vec(), | ||
tx_sequence_number: (first_tx + i) as i64, | ||
sender: ev.sender.to_vec(), | ||
}), | ||
); | ||
} | ||
|
||
Ok(values.into_iter().collect()) | ||
} | ||
|
||
async fn commit(values: &[Self::Value], conn: &mut db::Connection<'_>) -> Result<usize> { | ||
Ok(diesel::insert_into(ev_emit_pkg::table) | ||
.values(values) | ||
.on_conflict_do_nothing() | ||
.execute(conn) | ||
.await?) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use std::{collections::BTreeSet, sync::Arc}; | ||
|
||
use anyhow::{Context, Result}; | ||
use diesel_async::RunQueryDsl; | ||
use sui_types::full_checkpoint_content::CheckpointData; | ||
|
||
use crate::{db, models::events::StoredEvStructInst, schema::ev_struct_inst}; | ||
|
||
use super::Handler; | ||
|
||
pub struct EvStructInst; | ||
|
||
#[async_trait::async_trait] | ||
impl Handler for EvStructInst { | ||
const NAME: &'static str = "ev_struct_inst"; | ||
|
||
const BATCH_SIZE: usize = 100; | ||
const CHUNK_SIZE: usize = 1000; | ||
const MAX_PENDING_SIZE: usize = 10000; | ||
|
||
type Value = StoredEvStructInst; | ||
|
||
fn handle(checkpoint: &Arc<CheckpointData>) -> Result<Vec<Self::Value>> { | ||
let CheckpointData { | ||
transactions, | ||
checkpoint_summary, | ||
.. | ||
} = checkpoint.as_ref(); | ||
|
||
let mut values = BTreeSet::new(); | ||
let first_tx = checkpoint_summary.network_total_transactions as usize - transactions.len(); | ||
|
||
for (i, tx) in transactions.iter().enumerate() { | ||
let tx_sequence_number = (first_tx + i) as i64; | ||
for (j, ev) in tx.events.iter().flat_map(|evs| evs.data.iter().enumerate()) { | ||
values.insert(StoredEvStructInst { | ||
package: ev.type_.address.to_vec(), | ||
module: ev.type_.module.to_string(), | ||
name: ev.type_.name.to_string(), | ||
instantiation: bcs::to_bytes(&ev.type_.type_params) | ||
.with_context(|| format!( | ||
"Failed to serialize type parameters for event ({tx_sequence_number}, {j})" | ||
))?, | ||
tx_sequence_number: (first_tx + i) as i64, | ||
sender: ev.sender.to_vec(), | ||
}); | ||
} | ||
} | ||
|
||
Ok(values.into_iter().collect()) | ||
} | ||
|
||
async fn commit(values: &[Self::Value], conn: &mut db::Connection<'_>) -> Result<usize> { | ||
Ok(diesel::insert_into(ev_struct_inst::table) | ||
.values(values) | ||
.on_conflict_do_nothing() | ||
.execute(conn) | ||
.await?) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use std::{collections::BTreeSet, sync::Arc}; | ||
|
||
use anyhow::Result; | ||
use diesel_async::RunQueryDsl; | ||
use sui_types::full_checkpoint_content::CheckpointData; | ||
|
||
use crate::{db, models::events::StoredEvStructMod, schema::ev_struct_mod}; | ||
|
||
use super::Handler; | ||
|
||
pub struct EvStructMod; | ||
|
||
#[async_trait::async_trait] | ||
impl Handler for EvStructMod { | ||
const NAME: &'static str = "ev_struct_mod"; | ||
|
||
const BATCH_SIZE: usize = 100; | ||
const CHUNK_SIZE: usize = 1000; | ||
const MAX_PENDING_SIZE: usize = 10000; | ||
|
||
type Value = StoredEvStructMod; | ||
|
||
fn handle(checkpoint: &Arc<CheckpointData>) -> Result<Vec<Self::Value>> { | ||
let CheckpointData { | ||
transactions, | ||
checkpoint_summary, | ||
.. | ||
} = checkpoint.as_ref(); | ||
|
||
let mut values = BTreeSet::new(); | ||
let first_tx = checkpoint_summary.network_total_transactions as usize - transactions.len(); | ||
|
||
for (i, tx) in transactions.iter().enumerate() { | ||
values.extend( | ||
tx.events | ||
.iter() | ||
.flat_map(|evs| &evs.data) | ||
.map(|ev| StoredEvStructMod { | ||
package: ev.type_.address.to_vec(), | ||
module: ev.type_.module.to_string(), | ||
tx_sequence_number: (first_tx + i) as i64, | ||
sender: ev.sender.to_vec(), | ||
}), | ||
); | ||
} | ||
|
||
Ok(values.into_iter().collect()) | ||
} | ||
|
||
async fn commit(values: &[Self::Value], conn: &mut db::Connection<'_>) -> Result<usize> { | ||
Ok(diesel::insert_into(ev_struct_mod::table) | ||
.values(values) | ||
.on_conflict_do_nothing() | ||
.execute(conn) | ||
.await?) | ||
} | ||
} |
Oops, something went wrong.