Skip to content

Commit

Permalink
Use MoveStruct instead of SuiMoveStruct for wrapped indexing (#16348)
Browse files Browse the repository at this point in the history 
## Description 

The current approach to wrapped object indexing uses `SuiMoveStruct`
which hides some underlying types like `Balance` which causes it to miss
indexing such types. Instead, we will start using `MoveStruct` i.e. the
actual underlying type and retain the same functionality (i.e we will
continue to skip indexing string and url types which we were doing
before with `SuiMoveStruct` as well)

## Test Plan 

Added balance type in the test to check if things work as expected
  • Loading branch information
@sadhansood
sadhansood authored Feb 24, 2024
1 parent e7ee886 commit 840bde8
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 125 deletions.
226 changes: 102 additions & 124 deletions crates/sui-analytics-indexer/src/handlers/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@
use std::collections::{BTreeMap, BTreeSet};

use anyhow::{anyhow, Result};
use move_core_types::annotated_value::{MoveStruct, MoveTypeLayout};
use move_core_types::annotated_value::{MoveStruct, MoveTypeLayout, MoveValue};
use move_core_types::language_storage::{StructTag, TypeTag};

use sui_indexer::framework::Handler;
use sui_json_rpc_types::{SuiMoveStruct, SuiMoveValue};
use sui_package_resolver::{PackageStore, Resolver};
use sui_types::base_types::ObjectID;
use sui_types::effects::TransactionEffects;
Expand All @@ -30,6 +29,13 @@ pub mod transaction_handler;
pub mod transaction_objects_handler;
pub mod wrapped_object_handler;

const WRAPPED_INDEXING_DISALLOW_LIST: [&str; 4] = [
"0x1::string::String",
"0x1::ascii::String",
"0x2::url::Url",
"0x2::object::ID",
];

#[async_trait::async_trait]
pub trait AnalyticsHandler<S>: Handler {
/// Read back rows which are ready to be persisted. This function
Expand Down Expand Up @@ -181,163 +187,135 @@ pub struct WrappedStruct {
struct_tag: Option<StructTag>,
}

fn parse_struct_field(
fn parse_struct(
path: &str,
sui_move_value: SuiMoveValue,
curr_struct: &mut WrappedStruct,
move_struct: MoveStruct,
all_structs: &mut BTreeMap<String, WrappedStruct>,
) {
match sui_move_value {
SuiMoveValue::Struct(move_struct) => parse_struct(path, move_struct, all_structs),
SuiMoveValue::Vector(fields) => {
for (index, field) in fields.iter().enumerate() {
parse_struct_field(
&format!("{}[{}]", path, &index),
field.clone(),
curr_struct,
all_structs,
);
}
}
SuiMoveValue::Option(option_sui_move_value) => {
if option_sui_move_value.is_some() {
parse_struct_field(
path,
option_sui_move_value.unwrap(),
curr_struct,
all_structs,
);
}
}
SuiMoveValue::UID { id } => curr_struct.object_id = Some(id),
_ => {}
let mut wrapped_struct = WrappedStruct {
struct_tag: Some(move_struct.type_),
..Default::default()
};
for (k, v) in move_struct.fields {
parse_struct_field(
&format!("{}.{}", path, &k),
v,
&mut wrapped_struct,
all_structs,
);
}
all_structs.insert(path.to_string(), wrapped_struct);
}

fn parse_struct(
fn parse_struct_field(
path: &str,
sui_move_struct: SuiMoveStruct,
move_value: MoveValue,
curr_struct: &mut WrappedStruct,
all_structs: &mut BTreeMap<String, WrappedStruct>,
) {
let mut wrapped_struct = WrappedStruct::default();
match sui_move_struct {
SuiMoveStruct::WithTypes { type_, fields } => {
wrapped_struct.struct_tag = Some(type_);
for (k, v) in fields {
parse_struct_field(
&format!("{}.{}", path, &k),
v,
&mut wrapped_struct,
all_structs,
);
}
all_structs.insert(path.to_string(), wrapped_struct);
}
SuiMoveStruct::WithFields(fields) => {
for (k, v) in fields {
parse_struct_field(
&format!("{}.{}", path, &k),
v,
&mut wrapped_struct,
all_structs,
);
match move_value {
MoveValue::Struct(move_struct) => {
let values = move_struct
.fields
.iter()
.map(|(id, value)| (id.to_string(), value))
.collect::<BTreeMap<_, _>>();
let struct_name = format!(
"0x{}::{}::{}",
move_struct.type_.address.short_str_lossless(),
move_struct.type_.module,
move_struct.type_.name
);
if "0x2::object::UID" == struct_name {
if let Some(MoveValue::Struct(id_struct)) = values.get("id").cloned() {
let id_values = id_struct
.fields
.iter()
.map(|(id, value)| (id.to_string(), value))
.collect::<BTreeMap<_, _>>();
if let Some(MoveValue::Address(address) | MoveValue::Signer(address)) =
id_values.get("bytes").cloned()
{
curr_struct.object_id = Some(ObjectID::from_address(*address))
}
}
} else if "0x1::option::Option" == struct_name {
// Option in sui move is implemented as vector of size 1
if let Some(MoveValue::Vector(vec_values)) = values.get("vec").cloned() {
if let Some(first_value) = vec_values.first() {
parse_struct_field(
&format!("{}[0]", path),
first_value.clone(),
curr_struct,
all_structs,
);
}
}
} else if !WRAPPED_INDEXING_DISALLOW_LIST.contains(&&*struct_name) {
// Do not index most common struct types i.e. string, url, etc
parse_struct(path, move_struct, all_structs)
}
all_structs.insert(path.to_string(), wrapped_struct);
}
SuiMoveStruct::Runtime(values) => {
for (index, field) in values.iter().enumerate() {
MoveValue::Vector(fields) => {
for (index, field) in fields.iter().enumerate() {
parse_struct_field(
&format!("{}[{}]", path, &index),
field.clone(),
&mut wrapped_struct,
curr_struct,
all_structs,
);
}
all_structs.insert(path.to_string(), wrapped_struct);
}
_ => {}
}
}

#[cfg(test)]
mod tests {
use crate::handlers::parse_struct;
use move_core_types::account_address::AccountAddress;
use move_core_types::annotated_value::{MoveStruct, MoveValue};
use move_core_types::identifier::Identifier;
use move_core_types::language_storage::StructTag;
use std::collections::BTreeMap;
use sui_json_rpc_types::SuiMoveStruct;
use std::str::FromStr;
use sui_types::base_types::ObjectID;

#[tokio::test]
async fn test_wrapped_object_parsing_simple() -> anyhow::Result<()> {
let input = r#"{"x":{"y":{"id":{"id":"0x100"},"size":"15"},"id":{"id":"0x200"}},"id":{"id":"0x300"}}"#;
let move_struct: SuiMoveStruct = serde_json::from_str(input).unwrap();
let mut all_structs = BTreeMap::new();
parse_struct("$", move_struct, &mut all_structs);
assert_eq!(
all_structs.get("$").unwrap().object_id,
Some(ObjectID::from_hex_literal("0x300")?)
);
assert_eq!(
all_structs.get("$.x").unwrap().object_id,
Some(ObjectID::from_hex_literal("0x200")?)
);
assert_eq!(
all_structs.get("$.x.y").unwrap().object_id,
Some(ObjectID::from_hex_literal("0x100")?)
);
Ok(())
}

#[tokio::test]
async fn test_wrapped_object_parsing_with_array() -> anyhow::Result<()> {
let input = r#"{"ema_prices":{"id":{"id":"0x100"},"size":"0"},"id":{"id":"0x200"},"prices":{"id":{"id":"0x300"},"size":"11"},"primary_price_update_policy":{"id":{"id":"0x400"},"rules":{"contents":[{"name":"910f30cbc7f601f75a5141a01265cd47c62d468707c5e1aecb32a18f448cb25a::rule::Rule"}]}},"secondary_price_update_policy":{"id":{"id":"0x500"},"rules":{"contents":[]}}}"#;
let move_struct: SuiMoveStruct = serde_json::from_str(input).unwrap();
async fn test_wrapped_object_parsing() -> anyhow::Result<()> {
let uid_field = MoveValue::Struct(MoveStruct {
type_: StructTag::from_str("0x2::object::UID")?,
fields: vec![(
Identifier::from_str("id")?,
MoveValue::Struct(MoveStruct {
type_: StructTag::from_str("0x2::object::ID")?,
fields: vec![(
Identifier::from_str("bytes")?,
MoveValue::Signer(AccountAddress::from_hex_literal("0x300")?),
)],
}),
)],
});
let balance_field = MoveValue::Struct(MoveStruct {
type_: StructTag::from_str("0x2::balance::Balance")?,
fields: vec![(Identifier::from_str("value")?, MoveValue::U32(10))],
});
let move_struct = MoveStruct {
type_: StructTag::from_str("0x2::test::Test")?,
fields: vec![
(Identifier::from_str("id")?, uid_field),
(Identifier::from_str("principal")?, balance_field),
],
};
let mut all_structs = BTreeMap::new();
parse_struct("$", move_struct, &mut all_structs);
assert_eq!(
all_structs.get("$").unwrap().object_id,
Some(ObjectID::from_hex_literal("0x200")?)
);
assert_eq!(
all_structs.get("$.ema_prices").unwrap().object_id,
Some(ObjectID::from_hex_literal("0x100")?)
);
assert_eq!(
all_structs.get("$.prices").unwrap().object_id,
Some(ObjectID::from_hex_literal("0x300")?)
);
assert_eq!(
all_structs
.get("$.primary_price_update_policy")
.unwrap()
.object_id,
Some(ObjectID::from_hex_literal("0x400")?)
);
assert_eq!(
all_structs
.get("$.secondary_price_update_policy")
.unwrap()
.object_id,
Some(ObjectID::from_hex_literal("0x500")?)
);
assert_eq!(
all_structs
.get("$.secondary_price_update_policy.rules")
.unwrap()
.object_id,
None
);
assert_eq!(
all_structs
.get("$.primary_price_update_policy.rules")
.unwrap()
.object_id,
None
);
assert_eq!(
all_structs
.get("$.primary_price_update_policy.rules.contents[0]")
.unwrap()
.object_id,
None
all_structs.get("$.principal").unwrap().struct_tag,
Some(StructTag::from_str("0x2::balance::Balance")?)
);
Ok(())
}
Expand Down
2 changes: 1 addition & 1 deletion crates/sui-analytics-indexer/src/handlers/wrapped_object_handler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ impl WrappedObjectHandler {
};
let mut wrapped_structs = BTreeMap::new();
if let Some(move_struct) = move_struct {
parse_struct("$", move_struct.into(), &mut wrapped_structs);
parse_struct("$", move_struct, &mut wrapped_structs);
}
for (json_path, wrapped_struct) in wrapped_structs.iter() {
let entry = WrappedObjectEntry {
Expand Down

0 comments on commit 840bde8

Please sign in to comment.