diff --git a/.gitignore b/.gitignore index aa9cae59..323d985f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ Cargo.lock cli/rindexer.yaml cli/.env examples/* +!examples/rindexer_demo_cli documentation/node_modules documentation/dist documentation/docs/dist diff --git a/README.md b/README.md index 7c2c9f2f..99660cd8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 🦀 rindexer 🦀 -Note rindexer is brand new and only in beta and actively under development, things will change and bugs will exist - if you find any bugs or have any +Note rindexer is brand new and actively under development, things will change and bugs will exist - if you find any bugs or have any feature requests please open an issue on [github](https://github.com/joshstevens19/rindexer/issues). rindexer is an opensource powerful, high-speed indexing toolset developed in Rust, designed for compatibility with any EVM chain. @@ -39,6 +39,7 @@ Commands: add Add elements such as contracts to the rindexer.yaml file codegen Generates rust code based on rindexer.yaml or graphql queries delete Delete data from the postgres database or csv files + phantom Use phantom events to add your own events to contracts help Print this message or the help of the given subcommand(s) Options: @@ -111,6 +112,13 @@ you can run `cargo fmt` to format the code, rules have been mapped in the `rustf Anyone is welcome to contribute to rindexer, feel free to look over the issues or open a new one if you have any new ideas or bugs you have found. +### Playing around with the CLI locally + +You can use the `make` commands to run the CLI commands locally, this is useful for testing and developing. +These are located in the `cli` folder > `Makefile`. It uses `CURDIR` to resolve the paths for you, so they should work +out of the box. The examples repo has a `rindexer_demo_cli` folder which you can modify (please do not commit any changes though) +or spin up a new no-code project using the make commands. + ## Release To release a new rindexer you have to do a few things: diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0fc2912a..5cd12554 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -15,6 +15,8 @@ clap = { version = "4.4.11", features = ["derive"] } regex = "1.5.4" colored = "2.0" tokio = "1.35.1" +serde = { version = "1.0.204", features = ["derive"] } +serde_json = "1.0.120" # build jemallocator = { version = "0.5.0", optional = true } diff --git a/cli/Makefile b/cli/Makefile index 5764b074..a317c3e4 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -1,36 +1,43 @@ +prod_build: + RUSTFLAGS='-C target-cpu=native' cargo build --release --features jemalloc new_no_code: - cargo run -- new --path /Users/joshstevens/code/rindexer/examples no-code + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- new --path $(CURDIR)/../examples no-code new_rust: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- new --path /Users/joshstevens/code rust -start: - cargo run -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli all -start_prod: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start all + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- new --path $(CURDIR)/../../ rust start_indexer: - cargo run -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli indexer -start_indexer_base_paint: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/base_paint indexer -start_graphql_base_paint: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/base_paint graphql -start_indexer_prod: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli all -start_indexer_lens_mirrors: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/lens_mirrors all + RUSTFLAGS='-C target-cpu=native' RUST_BACKTRACE='full' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/rindexer_demo_cli indexer +start_all: + RUSTFLAGS='-C target-cpu=native' RUST_BACKTRACE='full' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/rindexer_demo_cli all start_graphql: - cargo run -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli graphql -start_indexer_uniswap_v3_factory: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/uniswap_v3_factory all -codegen: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path /Users/joshstevens/code/kami typings + RUSTFLAGS='-C target-cpu=native' RUST_BACKTRACE='full' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/rindexer_demo_cli graphql codegen_typings: - cargo run -- codegen typings + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path $(CURDIR)/../rindexer_rust_playground typings codegen_indexer: - cargo run -- codegen indexer + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path $(CURDIR)/../rindexer_rust_playground indexer codegen_graphql: - cargo run -- codegen --path /Users/joshstevens/code/rindexer/examples/base_paint graphql --endpoint http://0.0.0.0:5005/graphql + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path $(CURDIR)/../examples/rindexer_demo_cli graphql --endpoint http://0.0.0.0:5005/graphql add_contract: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- add --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli contract + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- add --path $(CURDIR)/../examples/rindexer_demo_cli contract delete: - RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- delete --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli -prod_build: - RUSTFLAGS='-C target-cpu=native' cargo build --release --features jemalloc \ No newline at end of file + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- delete --path $(CURDIR)/../examples/rindexer_demo_cli +phantom_init: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli init +phantom_clone: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli clone --contract-name RocketPoolETH --network ethereum +phantom_compile: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli compile --contract-name RocketPoolETH --network ethereum +phantom_deploy: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli deploy --contract-name RocketPoolETH --network ethereum +help: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --help + + +################################################################################ +# LOCAL NONE CHECKED IN PROJECT COMMANDS +################################################################################ +start_indexer_base_paint: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/base_paint indexer +start_graphql_base_paint: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/base_paint graphql +start_indexer_lens_mirrors: + RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/lens_mirrors all \ No newline at end of file diff --git a/cli/README.md b/cli/README.md index 5786f6fd..753af9d5 100644 --- a/cli/README.md +++ b/cli/README.md @@ -32,6 +32,7 @@ Commands: add Add elements such as contracts to the rindexer.yaml file codegen Generates rust code based on rindexer.yaml or graphql queries delete Delete data from the postgres database or csv files + phantom Use phantom events to add your own events to contracts help Print this message or the help of the given subcommand(s) Options: @@ -41,19 +42,10 @@ Options: ## Working with CLI locally -The best way to work with the CLI is to use the `cargo run` command with args after it inside the CLI project, -for example if I wanted to create a new project I would run: +The best way to work with the CLI is to use the `Makefile` predefined commands. -```bash -cargo run -- new --path PATH_TO_CREATE_PROJECT no-code -``` - -This would create a new no-code project in the path you specified. - -If you wanted to look at the help you can run: +You can also run your own commands using cargo run, example below would create a new no-code project in the path you specified. ```bash -cargo run -- help -``` - -This will show you all the commands available to you. \ No newline at end of file +cargo run -- new --path PATH_TO_CREATE_PROJECT no-code +``` \ No newline at end of file diff --git a/cli/src/cli_interface.rs b/cli/src/cli_interface.rs index 97088c72..4b81b7d8 100644 --- a/cli/src/cli_interface.rs +++ b/cli/src/cli_interface.rs @@ -1,4 +1,4 @@ -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; #[allow(clippy::upper_case_acronyms)] #[derive(Parser, Debug)] @@ -99,6 +99,24 @@ pub enum Commands { #[clap(long, short)] path: Option, }, + /// Use phantom events to add your own events to contracts + /// + /// This command helps you use phantom events within rindexer. + /// + /// Example: + /// `rindexer phantom init` or + /// `rindexer phantom clone --contract-name --network ` or + /// `rindexer phantom compile --contract-name --network ` or + /// `rindexer phantom deploy --contract-name --network ` + #[clap(name = "phantom")] + Phantom { + #[clap(subcommand)] + subcommand: PhantomSubcommands, + + /// optional - The path to create the project in, default will be where the command is run. + #[clap(long, short)] + path: Option, + }, } #[derive(Subcommand, Debug)] @@ -197,3 +215,77 @@ pub enum CodegenSubcommands { endpoint: Option, }, } + +#[derive(Args, Debug)] +pub struct PhantomBaseArgs { + /// The name of the contract + #[clap(value_parser)] + pub contract_name: String, + + /// The network the contract is on + #[clap(value_parser)] + pub network: String, +} + +#[derive(Subcommand, Debug)] +pub enum PhantomSubcommands { + /// Sets up phantom events on rindexer + /// + /// Want to add your own custom events to contracts? This command will help you do that. + /// + /// Example: + /// `rindexer phantom init` + #[clap(name = "init")] + Init, + + /// Clone the contract with the network you wish to add phantom events to. + /// + /// Note contract name and network are your values in your rindexer.yaml file. + /// + /// Example: + /// `rindexer phantom clone --contract-name --network ` + #[clap(name = "clone")] + Clone { + /// The name of the contract to clone + #[arg(long)] + contract_name: String, + + /// The network + #[arg(long)] + network: String, + }, + + /// Compiles the phantom contract + /// + /// Note contract name and network are your values in your rindexer.yaml file. + /// + /// Example: + /// `rindexer phantom compile --contract-name --network ` + #[clap(name = "compile")] + Compile { + /// The name of the contract to clone + #[arg(long)] + contract_name: String, + + /// The network + #[arg(long)] + network: String, + }, + + /// Deploy the modified phantom contract + /// + /// This will compile and update your rindexer project with the phantom events. + /// + /// Example: + /// `rindexer phantom deploy --contract-name --network ` + #[clap(name = "deploy")] + Deploy { + /// The name of the contract to clone + #[arg(long)] + contract_name: String, + + /// The network + #[arg(long)] + network: String, + }, +} diff --git a/cli/src/commands/add.rs b/cli/src/commands/add.rs index 6ba29c70..a300f352 100644 --- a/cli/src/commands/add.rs +++ b/cli/src/commands/add.rs @@ -8,9 +8,9 @@ use ethers_etherscan::Client; use rindexer::{ manifest::{ contract::{Contract, ContractDetails}, - yaml::{read_manifest, write_manifest, YAML_CONFIG_NAME}, + yaml::{read_manifest_raw, write_manifest, YAML_CONFIG_NAME}, }, - write_file, + public_read_env_value, write_file, }; use crate::{ @@ -28,7 +28,7 @@ pub async fn handle_add_contract_command( let rindexer_yaml_path = project_path.join(YAML_CONFIG_NAME); - let mut manifest = read_manifest(&rindexer_yaml_path).inspect_err(|e| { + let mut manifest = read_manifest_raw(&rindexer_yaml_path).inspect_err(|e| { print_error_message(&format!("Could not read the rindexer.yaml file: {}", e)) })?; @@ -62,15 +62,15 @@ pub async fn handle_add_contract_command( .1; let chain_network = Chain::try_from(chain_id) - .inspect_err(|_| print_error_message("Network is not supported by etherscan API"))?; + .inspect_err(|_| print_error_message("Network is not supported by etherscan API, please add the contract manually in the rindexer.yaml file"))?; let contract_address = prompt_for_input(&format!("Enter {} Contract Address", network), None, None, None); - let etherscan_api_key = manifest - .global - .as_ref() - .and_then(|global| global.etherscan_api_key.as_ref()) - .map_or(BACKUP_ETHERSCAN_API_KEY, String::as_str); + let etherscan_api_key = + manifest.global.as_ref().and_then(|global| global.etherscan_api_key.as_ref()).map_or_else( + || BACKUP_ETHERSCAN_API_KEY.to_string(), + |key| public_read_env_value(key).unwrap_or_else(|_| key.to_string()), + ); let client = Client::builder() .with_api_key(etherscan_api_key) diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs index 9e73a637..9cc50811 100644 --- a/cli/src/commands/mod.rs +++ b/cli/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod add; pub mod codegen; pub mod delete; pub mod new; +pub mod phantom; pub mod start; const BACKUP_ETHERSCAN_API_KEY: &str = "DHBPB1EJ84JMSWP7C86387NK7IIRRQJVV1"; diff --git a/cli/src/commands/new.rs b/cli/src/commands/new.rs index 2cdcbe5f..5f90d3e5 100644 --- a/cli/src/commands/new.rs +++ b/cli/src/commands/new.rs @@ -152,6 +152,7 @@ pub fn handle_new_command( reorg_safe_distance: None, generate_csv: None, }], + phantom: None, global: None, storage: Storage { postgres: if postgres_enabled { diff --git a/cli/src/commands/phantom.rs b/cli/src/commands/phantom.rs new file mode 100644 index 00000000..0dc80609 --- /dev/null +++ b/cli/src/commands/phantom.rs @@ -0,0 +1,583 @@ +use std::{ + env, + error::Error, + fs, + fs::OpenOptions, + io::Write, + path::{Path, PathBuf}, + process::Command, +}; + +use ethers::types::{Address, ValueOrArray, U64}; +use rindexer::{ + manifest::{ + network::Network, + phantom::{Phantom, PhantomDyrpc, PhantomShadow}, + yaml::{read_manifest, read_manifest_raw, write_manifest, YAML_CONFIG_NAME}, + }, + phantom::{ + common::{read_compiled_contract, read_contract_clone_metadata}, + create_dyrpc_api_key, deploy_dyrpc_contract, + shadow::deploy_shadow_contract, + }, + public_read_env_value, write_file, +}; + +use crate::{ + cli_interface::{PhantomBaseArgs, PhantomSubcommands}, + commands::BACKUP_ETHERSCAN_API_KEY, + console::{ + print_error_message, print_success_message, print_warn_message, prompt_for_input, + prompt_for_input_list, + }, + rindexer_yaml::validate_rindexer_yaml_exist, +}; + +const RINDEXER_PHANTOM_API_ENV_KEY: &str = "RINDEXER_PHANTOM_API_KEY"; + +pub async fn handle_phantom_commands( + project_path: PathBuf, + command: &PhantomSubcommands, +) -> Result<(), Box> { + validate_rindexer_yaml_exist(&project_path); + + match command { + PhantomSubcommands::Init => handle_phantom_init(&project_path).await, + PhantomSubcommands::Clone { contract_name, network } => handle_phantom_clone( + &project_path, + &PhantomBaseArgs { + contract_name: contract_name.to_owned(), + network: network.to_owned(), + }, + ), + PhantomSubcommands::Compile { contract_name, network } => handle_phantom_compile( + &project_path, + &PhantomBaseArgs { + contract_name: contract_name.to_owned(), + network: network.to_owned(), + }, + ), + PhantomSubcommands::Deploy { contract_name, network } => { + handle_phantom_deploy( + &project_path, + &PhantomBaseArgs { + contract_name: contract_name.to_owned(), + network: network.to_owned(), + }, + ) + .await + } + } +} + +fn install_foundry() -> Result<(), Box> { + let foundry_check = + Command::new("which").arg("foundryup").output().expect("Failed to execute command"); + + if foundry_check.status.success() { + Ok(()) + } else { + println!("Foundry is not installed. Installing Foundry..."); + + let install_command = Command::new("sh") + .arg("-c") + .arg("curl -L https://foundry.paradigm.xyz | bash") + .status() + .map_err(|e| e.to_string())?; + + if install_command.success() { + Ok(()) + } else { + Err("Failed to install Foundry.".into()) + } + } +} + +async fn handle_phantom_init(project_path: &Path) -> Result<(), Box> { + let env_file = project_path.join(".env"); + let rindexer_yaml_path = project_path.join(YAML_CONFIG_NAME); + + let mut manifest = read_manifest_raw(&rindexer_yaml_path).inspect_err(|e| { + print_error_message(&format!("Could not read the rindexer.yaml file: {}", e)) + })?; + + if manifest.phantom.is_some() { + let error_message = "phantom already setup in rindexer.yaml"; + print_error_message(error_message); + return Err(error_message.into()); + } + + print_success_message("setting up phantom events on rindexer..."); + + install_foundry()?; + + let phantom_provider_choice = prompt_for_input_list( + "Which provider are you using?", + &["shadow".to_string(), "dyrpc".to_string()], + None, + ); + + let mut api_key_value = prompt_for_input( + if phantom_provider_choice == "dyrpc" { + "Enter your dyRPC API key (enter to new to generate a new key)" + } else { + "Enter your Shadow API key" + }, + None, + None, + None, + ); + + match phantom_provider_choice.as_str() { + "dyrpc" => { + if api_key_value == "new" { + api_key_value = create_dyrpc_api_key().await?; + println!( + "Your API has been created and key is {} - it has also been written to your .env file.", + api_key_value + ); + } + + manifest.phantom = Some(Phantom { + dyrpc: Some(PhantomDyrpc { + api_key: format!("${{{}}}", RINDEXER_PHANTOM_API_ENV_KEY), + }), + shadow: None, + }); + + write_manifest(&manifest, &rindexer_yaml_path)?; + } + "shadow" => { + let fork_id = prompt_for_input("Enter the fork ID", None, None, None); + + manifest.phantom = Some(Phantom { + shadow: Some(PhantomShadow { + api_key: format!("${{{}}}", RINDEXER_PHANTOM_API_ENV_KEY), + fork_id, + }), + dyrpc: None, + }); + + write_manifest(&manifest, &rindexer_yaml_path)?; + } + value => panic!("Unknown phantom provider: {}", value), + } + + let env_content = fs::read_to_string(&env_file).unwrap_or_default(); + + let value = api_key_value; + + let mut lines: Vec = env_content.lines().map(|line| line.to_string()).collect(); + let mut key_found = false; + for line in &mut lines { + if line.starts_with(&format!("{}=", RINDEXER_PHANTOM_API_ENV_KEY)) { + *line = format!("{}={}", RINDEXER_PHANTOM_API_ENV_KEY, value); + key_found = true; + break; + } + } + + if !key_found { + lines.push(format!("{}={}", RINDEXER_PHANTOM_API_ENV_KEY, value)); + } + + let new_env_content = lines.join("\n"); + + let mut file = OpenOptions::new().write(true).truncate(true).create(true).open(&env_file)?; + + writeln!(file, "{}", new_env_content)?; + + print_success_message("rindexer Phantom events are now setup.\nYou can now use `rindexer phantom clone --contract-name --network ` to start adding your own custom events."); + + Ok(()) +} + +fn forge_clone_contract( + clone_in: &Path, + network: &Network, + address: &Address, + contract_name: &str, + etherscan_api_key: &str, +) -> Result<(), Box> { + print_success_message(&format!( + "Cloning contract {} on network {} at address {:?} this may take a little moment...", + contract_name, network.name, address + )); + let output = Command::new("forge") + .arg("clone") + .arg("--no-commit") + .arg(format!("{:?}", address)) + //.arg(format!("--chain {}", network.chain_id)) + .arg("--etherscan-api-key") + .arg(etherscan_api_key) + .arg(contract_name) + .current_dir(clone_in) + .output()?; + + if output.status.success() { + Ok(()) + } else { + print_error_message(&format!( + "Failed to clone contract: {} at address: {:?}", + contract_name, address + )); + print_error_message(&format!("Error: {}", String::from_utf8_lossy(&output.stderr))); + Err("Failed to clone contract".into()) + } +} + +fn handle_phantom_clone(project_path: &Path, args: &PhantomBaseArgs) -> Result<(), Box> { + let rindexer_yaml_path = project_path.join(YAML_CONFIG_NAME); + + let manifest = read_manifest(&rindexer_yaml_path).inspect_err(|e| { + print_error_message(&format!("Could not read the rindexer.yaml file: {}", e)) + })?; + + if manifest.phantom.is_none() { + let error_message = + "phantom not setup in rindexer.yaml. Please run `rindexer phantom init` first."; + print_error_message(error_message); + return Err(error_message.into()); + } + + let cloning_location = + project_path.join("phantom").join(&args.network).join(args.contract_name.as_str()); + if cloning_location.exists() { + let error_message = format!("Phantom contract {} on network {} already cloned in {}. If you want to clone it again please delete the folder first.", args.contract_name, args.network, cloning_location.display()); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let contract = manifest.contracts.iter().find(|c| c.name == args.contract_name); + match contract { + Some(contract) => { + let network = manifest.networks.iter().find(|n| n.name == args.network); + if network.is_none() { + let error_message = format!("Network {} not found in rindexer.yaml", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + if network.unwrap().chain_id != 1 { + let error_message = format!("Network {} is not supported", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let contract_network = contract.details.iter().find(|c| c.network == args.network); + if let Some(contract_network) = contract_network { + if let Some(address) = contract_network.address() { + // pick the first one as the ABI has to match so assume all contracts do + let address = match address { + ValueOrArray::Value(address) => address, + ValueOrArray::Array(addresses) => { + print_warn_message(&format!("Multiple addresses found for contract {} on network {} rindexer.yaml, using first one", args.contract_name.as_str(), args.network.as_str())); + addresses.first().unwrap() + } + }; + + if !project_path.join("phantom").exists() { + fs::create_dir(project_path.join("phantom"))?; + } + + let clone_in = project_path.join("phantom").join(&args.network); + if !clone_in.exists() { + fs::create_dir(&clone_in)?; + } + + let etherscan_api_key = manifest + .global + .as_ref() + .and_then(|global| global.etherscan_api_key.as_ref()) + .map_or_else( + || BACKUP_ETHERSCAN_API_KEY.to_string(), + |key| public_read_env_value(key).unwrap_or_else(|_| key.to_string()), + ); + + forge_clone_contract( + &clone_in, + network.unwrap(), + address, + contract.name.as_str(), + ðerscan_api_key, + ) + .map_err(|e| format!("Failed to clone contract: {}", e))?; + + print_success_message(format!("\ncloned {} in {} you can start adding your custom events.\nYou can now use `rindexer phantom compile -contract-name {} --network {}` to compile the phantom contract anytime.", contract.name.as_str(), clone_in.display(), contract.name.as_str(), args.network).as_str()); + + Ok(()) + } else { + let error_message = format!( + "Contract {} in network {} does not have an address in rindexer.yaml", + args.contract_name, args.network + ); + print_error_message(&error_message); + Err(error_message.into()) + } + } else { + let error_message = format!( + "Network {} not found in contract {} in rindexer.yaml", + args.network, args.contract_name + ); + print_error_message(&error_message); + Err(error_message.into()) + } + } + None => { + let error_message = + format!("Contract {} not found in rindexer.yaml", args.contract_name); + print_error_message(&error_message); + Err(error_message.into()) + } + } +} + +fn forge_compile_contract( + compile_in: &Path, + network: &Network, + contract_name: &str, +) -> Result<(), Box> { + print_success_message(&format!( + "Compiling contract {} on network {}...", + contract_name, network.name + )); + let output = Command::new("forge").arg("build").current_dir(compile_in).output()?; + + if output.status.success() { + Ok(()) + } else { + print_error_message(&format!( + "Failed to compile contract: {} for network: {}", + contract_name, network.name + )); + print_error_message(&format!("Error: {}", String::from_utf8_lossy(&output.stderr))); + Err("Failed to compile contract".into()) + } +} + +fn get_phantom_network_name(args: &PhantomBaseArgs) -> String { + format!("phantom_{}_{}", args.network, args.contract_name) +} + +fn handle_phantom_compile( + project_path: &Path, + args: &PhantomBaseArgs, +) -> Result<(), Box> { + let rindexer_yaml_path = project_path.join(YAML_CONFIG_NAME); + + let manifest = read_manifest(&rindexer_yaml_path).inspect_err(|e| { + print_error_message(&format!("Could not read the rindexer.yaml file: {}", e)) + })?; + + if manifest.phantom.is_none() { + let error_message = + "phantom not setup in rindexer.yaml. Please run `rindexer phantom init` first."; + print_error_message(error_message); + return Err(error_message.into()); + } + + if !project_path.join("phantom").exists() { + let error_message = + "phantom folder not found in the project. Please run `rindexer phantom init` first."; + print_error_message(error_message); + return Err(error_message.into()); + } + + let network_path = project_path.join("phantom").join(&args.network); + if !network_path.exists() { + let error_message = format!("phantom network {} folder not found in the project. Please run `rindexer phantom clone` first.", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let compile_in = network_path.join(args.contract_name.as_str()); + if !compile_in.exists() { + let error_message = format!("phantom contract {} folder not found in the project. Please run `rindexer phantom clone` first.", args.contract_name); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let contract = manifest.contracts.iter().find(|c| c.name == args.contract_name); + match contract { + Some(contract) => { + let name = get_phantom_network_name(args); + let network = + manifest.networks.iter().find(|n| n.name == args.network || n.name == name); + if network.is_none() { + let error_message = format!("Network {} not found in rindexer.yaml", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + if network.unwrap().chain_id != 1 { + let error_message = format!("Network {} is not supported", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let contract_network = + contract.details.iter().find(|c| c.network == args.network || c.network == name); + if contract_network.is_some() { + forge_compile_contract(&compile_in, network.unwrap(), &args.contract_name) + .map_err(|e| format!("Failed to compile contract: {}", e))?; + + print_success_message(format!("\ncompiled contract {} for network {} successful.\nYou can use `rindexer phantom deploy --contract-name {} --network {}` to deploy the phantom contract and start indexing your custom events.", args.contract_name, args.network, args.contract_name, args.network).as_str()); + Ok(()) + } else { + let error_message = format!( + "Network {} not found in contract {} in rindexer.yaml", + args.network, args.contract_name + ); + print_error_message(&error_message); + Err(error_message.into()) + } + } + None => { + let error_message = + format!("Contract {} not found in rindexer.yaml", args.contract_name); + print_error_message(&error_message); + Err(error_message.into()) + } + } +} + +async fn handle_phantom_deploy( + project_path: &Path, + args: &PhantomBaseArgs, +) -> Result<(), Box> { + let rindexer_yaml_path = project_path.join(YAML_CONFIG_NAME); + + let mut manifest = read_manifest_raw(&rindexer_yaml_path).inspect_err(|e| { + print_error_message(&format!("Could not read the rindexer.yaml file: {}", e)) + })?; + + if manifest.phantom.is_none() { + let error_message = + "phantom not setup in rindexer.yaml. Please run `rindexer phantom init` first."; + print_error_message(error_message); + return Err(error_message.into()); + } + + if !project_path.join("phantom").exists() { + let error_message = + "phantom folder not found in the project. Please run `rindexer phantom init` first."; + print_error_message(error_message); + return Err(error_message.into()); + } + + let network_path = project_path.join("phantom").join(&args.network); + if !network_path.exists() { + let error_message = format!("phantom network {} folder not found in the project. Please run `rindexer phantom clone` first.", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let deploy_in = network_path.join(args.contract_name.as_str()); + if !deploy_in.exists() { + let error_message = format!("phantom contract {} folder not found in the project. Please run `rindexer phantom clone` first.", args.contract_name); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let contract = manifest.contracts.iter_mut().find(|c| c.name == args.contract_name); + match contract { + Some(contract) => { + let name = get_phantom_network_name(args); + let network = + manifest.networks.iter().find(|n| n.name == args.network || n.name == name); + if network.is_none() { + let error_message = format!("Network {} not found in rindexer.yaml", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + if network.unwrap().chain_id != 1 { + let error_message = format!("Network {} is not supported", args.network); + print_error_message(&error_message); + return Err(error_message.into()); + } + + let contract_network = contract + .details + .iter_mut() + .find(|c| c.network == args.network || c.network == name); + if contract_network.is_some() { + let clone_meta = read_contract_clone_metadata(&deploy_in)?; + + let phantom = manifest.phantom.as_ref().expect("Failed to get phantom"); + let rpc_url = if phantom.dyrpc_enabled() { + // only compile here as shadow has to do its own compiling to deploy + forge_compile_contract(&deploy_in, network.unwrap(), &args.contract_name) + .map_err(|e| format!("Failed to compile contract: {}", e))?; + + deploy_dyrpc_contract( + &env::var(RINDEXER_PHANTOM_API_ENV_KEY) + .expect("Failed to get phantom api key"), + &clone_meta, + &read_compiled_contract(&deploy_in, &clone_meta)?, + ) + .await + .map_err(|e| format!("Failed to deploy contract: {}", e))? + } else { + println!("deploying shadow contracts, this may take a while...."); + deploy_shadow_contract( + &env::var(RINDEXER_PHANTOM_API_ENV_KEY) + .expect("Failed to get phantom api key"), + &deploy_in, + &clone_meta, + phantom.shadow.as_ref().expect("Failed to get phantom shadow"), + ) + .await + .map_err(|e| format!("Failed to deploy contract: {}", e))? + }; + + let network_index = manifest.networks.iter().position(|net| net.name == name); + + if let Some(index) = network_index { + let net = &mut manifest.networks[index]; + net.rpc = rpc_url.to_string(); + } else { + manifest.networks.push(Network { + name: name.to_string(), + chain_id: network.unwrap().chain_id, + rpc: rpc_url.to_string(), + compute_units_per_second: None, + max_block_range: if phantom.dyrpc_enabled() { + Some(U64::from(20_000)) + } else { + Some(U64::from(2_000)) + }, + }); + } + + let compiled_contract = read_compiled_contract(&deploy_in, &clone_meta)?; + let abi_path = project_path.join("abis").join(format!("{}.abi.json", name)); + write_file( + &abi_path, + serde_json::to_string_pretty(&compiled_contract.abi).unwrap().as_str(), + )?; + + contract.abi = format!("./abis/{}.abi.json", name); + contract_network.unwrap().network = name; + + write_manifest(&manifest, &rindexer_yaml_path)?; + + print_success_message(format!("\ndeployed contract {} for network {} successful.\nYou can use `rindexer start all` to start indexing the phantom contract", args.contract_name, args.network).as_str()); + Ok(()) + } else { + let error_message = format!( + "Network {} not found in contract {} in rindexer.yaml", + args.network, args.contract_name + ); + print_error_message(&error_message); + Err(error_message.into()) + } + } + None => { + let error_message = + format!("Contract {} not found in rindexer.yaml", args.contract_name); + print_error_message(&error_message); + Err(error_message.into()) + } + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 3e1d72e5..180e3cea 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -21,7 +21,8 @@ use crate::{ cli_interface::{AddSubcommands, Commands, NewSubcommands, CLI}, commands::{ add::handle_add_contract_command, codegen::handle_codegen_command, - delete::handle_delete_command, new::handle_new_command, start::start, + delete::handle_delete_command, new::handle_new_command, phantom::handle_phantom_commands, + start::start, }, console::print_error_message, }; @@ -110,5 +111,10 @@ async fn main() -> Result<(), Box> { load_env_from_path(&resolved_path); handle_delete_command(resolved_path).await } + Commands::Phantom { subcommand, path } => { + let resolved_path = resolve_path(path).inspect_err(|e| print_error_message(e))?; + load_env_from_path(&resolved_path); + handle_phantom_commands(resolved_path, subcommand).await + } } } diff --git a/cli/src/rindexer_yaml.rs b/cli/src/rindexer_yaml.rs index cbcafb29..2ee2a2d8 100644 --- a/cli/src/rindexer_yaml.rs +++ b/cli/src/rindexer_yaml.rs @@ -1,18 +1,18 @@ -use std::{fs, path::PathBuf}; +use std::{fs, path::Path}; use rindexer::manifest::yaml::YAML_CONFIG_NAME; use crate::console::print_error_message; -pub fn rindexer_yaml_exists(project_path: &PathBuf) -> bool { +pub fn rindexer_yaml_exists(project_path: &Path) -> bool { fs::metadata(project_path.join(YAML_CONFIG_NAME)).is_ok() } -pub fn rindexer_yaml_does_not_exist(project_path: &PathBuf) -> bool { +pub fn rindexer_yaml_does_not_exist(project_path: &Path) -> bool { !rindexer_yaml_exists(project_path) } -pub fn validate_rindexer_yaml_exist(project_path: &PathBuf) { +pub fn validate_rindexer_yaml_exist(project_path: &Path) { if rindexer_yaml_does_not_exist(project_path) { print_error_message("rindexer.yaml does not exist in the current directory. Please use rindexer new to create a new project."); std::process::exit(1); diff --git a/core/Cargo.toml b/core/Cargo.toml index 50708094..c221486a 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -13,6 +13,7 @@ mockito = "0.30" [dependencies] ethers = { version = "2.0", features = ["rustls", "openssl"] } +ethers-solc = "2.0.14" tokio = { version = "1", features = ["full"] } tokio-postgres = { version="0.7", features=["with-uuid-1"] } bb8 = "0.8.3" @@ -42,7 +43,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "time"] chrono = "0.4.38" log = "0.4.20" colored = "2.0" -reqwest = { version = "0.12", features = ["json"] } +reqwest = { version = "0.11.27", features = ["json"] } thread_local = "1.1" native-tls = "0.2" postgres-native-tls = "0.5" diff --git a/core/src/abi.rs b/core/src/abi.rs index f1c6fff8..97c5edc4 100644 --- a/core/src/abi.rs +++ b/core/src/abi.rs @@ -15,10 +15,15 @@ use crate::{ #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ABIInput { + #[serde(default, skip_serializing_if = "Option::is_none")] pub indexed: Option, + pub name: String, + #[serde(rename = "type")] pub type_: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] pub components: Option>, } @@ -141,8 +146,10 @@ impl ABIInput { pub struct ABIItem { #[serde(default)] pub inputs: Vec, + #[serde(default)] pub name: String, + #[serde(rename = "type", default)] pub type_: String, } diff --git a/core/src/database/postgres/generate.rs b/core/src/database/postgres/generate.rs index 4910fad2..bee529ff 100644 --- a/core/src/database/postgres/generate.rs +++ b/core/src/database/postgres/generate.rs @@ -44,11 +44,17 @@ fn generate_event_table_sql(abi_inputs: &[EventInfo], schema_name: &str) -> Stri .map(|event_info| { let table_name = format!("{}.{}", schema_name, camel_to_snake(&event_info.name)); info!("Creating table if not exists: {}", table_name); + let event_columns = if event_info.inputs.is_empty() { + "".to_string() + } else { + generate_columns_with_data_types(&event_info.inputs).join(", ") + "," + }; + format!( "CREATE TABLE IF NOT EXISTS {} (\ rindexer_id SERIAL PRIMARY KEY NOT NULL, \ contract_address CHAR(66) NOT NULL, \ - {}, \ + {} \ tx_hash CHAR(66) NOT NULL, \ block_number NUMERIC NOT NULL, \ block_hash CHAR(66) NOT NULL, \ @@ -56,8 +62,7 @@ fn generate_event_table_sql(abi_inputs: &[EventInfo], schema_name: &str) -> Stri tx_index NUMERIC NOT NULL, \ log_index VARCHAR(78) NOT NULL\ );", - table_name, - generate_columns_with_data_types(&event_info.inputs).join(", ") + table_name, event_columns ) }) .collect::>() @@ -195,6 +200,7 @@ pub fn drop_tables_for_indexer_sql(project_path: &Path, indexer: &Indexer) -> Co Code::new(sql) } +#[allow(clippy::manual_strip)] pub fn solidity_type_to_db_type(abi_type: &str) -> String { let is_array = abi_type.ends_with("[]"); let base_type = abi_type.trim_end_matches("[]"); diff --git a/core/src/generator/networks_bindings.rs b/core/src/generator/networks_bindings.rs index 993640b6..69683891 100644 --- a/core/src/generator/networks_bindings.rs +++ b/core/src/generator/networks_bindings.rs @@ -15,7 +15,7 @@ pub fn network_provider_fn_name(network: &Network) -> String { fn generate_network_lazy_provider_code(network: &Network) -> Code { Code::new(format!( r#" - static ref {network_name}: Arc = create_client(&public_read_env_value("{network_url}").unwrap_or("{network_url}".to_string()), {compute_units_per_second}, {max_block_range}).expect("Error creating provider"); + static ref {network_name}: Arc = {client_fn}(&public_read_env_value("{network_url}").unwrap_or("{network_url}".to_string()), {compute_units_per_second}, {max_block_range} {placeholder_headers}).expect("Error creating provider"); "#, network_name = network_provider_name(network), network_url = network.rpc, @@ -30,6 +30,13 @@ fn generate_network_lazy_provider_code(network: &Network) -> Code { } else { "None".to_string() }, + client_fn = + if network.rpc.contains("shadow") { "create_shadow_client" } else { "create_client" }, + placeholder_headers = if network.rpc.contains("shadow") { + "" + } else { + ", HeaderMap::new()" + }, )) } @@ -88,10 +95,24 @@ pub fn generate_networks_code(networks: &[Network]) -> Code { use ethers::types::U64; use rindexer::{ lazy_static, - provider::{create_client, JsonRpcCachedProvider}, - public_read_env_value, + provider::{create_client, JsonRpcCachedProvider, RetryClientError}, + public_read_env_value, HeaderMap, }; use std::sync::Arc; + + #[allow(dead_code)] + fn create_shadow_client( + rpc_url: &str, + compute_units_per_second: Option, + max_block_range: Option, + ) -> Result, RetryClientError> { + let mut header = HeaderMap::new(); + header.insert( + "X-SHADOW-API-KEY", + public_read_env_value("RINDEXER_PHANTOM_API_KEY").unwrap().parse().unwrap(), + ); + create_client(rpc_url, compute_units_per_second, max_block_range, header) + } lazy_static! { "# diff --git a/core/src/indexer/fetch_logs.rs b/core/src/indexer/fetch_logs.rs index ff2eb53a..952b217f 100644 --- a/core/src/indexer/fetch_logs.rs +++ b/core/src/indexer/fetch_logs.rs @@ -206,6 +206,12 @@ async fn fetch_historic_logs_stream( } if logs_empty { + info!( + "{} - No events found between blocks {} - {}", + info_log_name, + from_block, + to_block + ); let next_from_block = to_block + 1; return if next_from_block > snapshot_to_block { None diff --git a/core/src/lib.rs b/core/src/lib.rs index bff817f5..bbe3f1aa 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -23,7 +23,9 @@ pub use api::{generate_graphql_queries, GraphqlOverrideSettings}; mod logger; pub use logger::setup_info_logger; mod abi; +pub use abi::ABIItem; pub mod event; +pub mod phantom; pub mod provider; mod start; mod types; @@ -33,6 +35,7 @@ pub use async_trait::async_trait; pub use colored::Colorize as RindexerColorize; pub use futures::FutureExt; pub use lazy_static::lazy_static; +pub use reqwest::header::HeaderMap; pub use start::{ start_rindexer, start_rindexer_no_code, IndexerNoCodeDetails, IndexingDetails, StartDetails, StartNoCodeDetails, diff --git a/core/src/manifest/core.rs b/core/src/manifest/core.rs index 1e5238d2..b70b5fb4 100644 --- a/core/src/manifest/core.rs +++ b/core/src/manifest/core.rs @@ -6,7 +6,7 @@ use crate::{ indexer::Indexer, manifest::{ contract::Contract, global::Global, graphql::GraphQLSettings, network::Network, - storage::Storage, + phantom::Phantom, storage::Storage, }, }; @@ -68,6 +68,9 @@ pub struct Manifest { pub contracts: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub phantom: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub global: Option, @@ -94,6 +97,16 @@ impl Manifest { self.storage.csv_enabled() && contract_csv_enabled } + + pub fn get_custom_headers(&self) -> reqwest::header::HeaderMap { + let mut headers = reqwest::header::HeaderMap::new(); + if let Some(phantom) = &self.phantom { + if let Some(shadow) = &phantom.shadow { + headers.insert("X-SHADOW-API-KEY", shadow.api_key.parse().unwrap()); + } + } + headers + } } pub fn deserialize_option_u64_from_string<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/core/src/manifest/mod.rs b/core/src/manifest/mod.rs index c838fbaa..fb4f35bc 100644 --- a/core/src/manifest/mod.rs +++ b/core/src/manifest/mod.rs @@ -3,5 +3,6 @@ pub mod core; pub mod global; pub mod graphql; pub mod network; +pub mod phantom; pub mod storage; pub mod yaml; diff --git a/core/src/manifest/phantom.rs b/core/src/manifest/phantom.rs new file mode 100644 index 00000000..aefbc035 --- /dev/null +++ b/core/src/manifest/phantom.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct PhantomShadow { + pub api_key: String, + pub fork_id: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct PhantomDyrpc { + pub api_key: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Phantom { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dyrpc: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub shadow: Option, +} + +impl Phantom { + pub fn dyrpc_enabled(&self) -> bool { + self.dyrpc.is_some() + } + + pub fn shadow_enabled(&self) -> bool { + self.shadow.is_some() + } +} diff --git a/core/src/manifest/yaml.rs b/core/src/manifest/yaml.rs index 8555d549..0e9ccefb 100644 --- a/core/src/manifest/yaml.rs +++ b/core/src/manifest/yaml.rs @@ -170,6 +170,24 @@ pub enum ReadManifestError { NoProjectPathFoundUsingParentOfManifestPath, } +pub fn read_manifest_raw(file_path: &PathBuf) -> Result { + let mut file = File::open(file_path)?; + let mut contents = String::new(); + + file.read_to_string(&mut contents)?; + + let manifest: Manifest = serde_yaml::from_str(&contents)?; + + let project_path = file_path.parent(); + match project_path { + None => Err(ReadManifestError::NoProjectPathFoundUsingParentOfManifestPath), + Some(project_path) => { + validate_manifest(project_path, &manifest)?; + Ok(manifest) + } + } +} + pub fn read_manifest(file_path: &PathBuf) -> Result { let mut file = File::open(file_path)?; let mut contents = String::new(); diff --git a/core/src/phantom/common.rs b/core/src/phantom/common.rs new file mode 100644 index 00000000..09f144f5 --- /dev/null +++ b/core/src/phantom/common.rs @@ -0,0 +1,66 @@ +use std::{error::Error, fs::File, io::Read, path::Path}; + +use ethers::abi::Abi; +use serde::Deserialize; + +#[derive(Deserialize, Debug)] +pub struct CloneMeta { + pub path: String, + + #[serde(rename = "targetContract")] + pub target_contract: String, + + pub address: String, + + #[serde(rename = "constructorArguments")] + pub constructor_arguments: String, +} + +impl CloneMeta { + fn get_out_contract_sol_from_path(&self) -> String { + self.path.split('/').last().unwrap_or_default().to_string() + } +} + +pub fn read_contract_clone_metadata(contract_path: &Path) -> Result> { + let meta_file_path = contract_path.join(".clone.meta"); + + let mut file = File::open(meta_file_path)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + let clone_meta: CloneMeta = serde_json::from_str(&contents)?; + + Ok(clone_meta) +} + +#[derive(Deserialize, Debug)] +pub struct Bytecode { + pub object: String, +} + +#[derive(Deserialize, Debug)] +pub struct CompiledContract { + pub abi: Abi, + + pub bytecode: Bytecode, +} + +pub fn read_compiled_contract( + contract_path: &Path, + clone_meta: &CloneMeta, +) -> Result> { + let compiled_file_path = contract_path.join("out").join(format!( + "{}/{}.json", + clone_meta.get_out_contract_sol_from_path(), + clone_meta.target_contract + )); + + let mut file = File::open(compiled_file_path)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + let compiled_contract: CompiledContract = serde_json::from_str(&contents)?; + + Ok(compiled_contract) +} diff --git a/core/src/phantom/dyrpc.rs b/core/src/phantom/dyrpc.rs new file mode 100644 index 00000000..1e5333a9 --- /dev/null +++ b/core/src/phantom/dyrpc.rs @@ -0,0 +1,127 @@ +use std::error::Error; + +use regex::Regex; +use reqwest::Client; +use serde::{Deserialize, Serialize}; + +use crate::phantom::common::{CloneMeta, CompiledContract}; + +#[derive(Serialize, Debug)] +struct DeployDyrpcRequest<'a> { + overlays: std::collections::HashMap<&'a str, DeployDyrpcDetails<'a>>, +} + +#[derive(Serialize, Debug)] +struct DeployDyrpcDetails<'a> { + #[serde(rename = "creationCode")] + creation_code: &'a str, + + #[serde(rename = "constructorArgs")] + constructor_args: &'a str, +} + +#[derive(Deserialize)] +pub struct DeployDyrpcContractResponse { + // don't care for now about these + // pub message: String, + // + // #[serde(rename = "overlayHash")] + // pub overlay_hash: String, + // + // pub addresses: Vec, + #[serde(rename = "overlayRpcUrl")] + pub rpc_url: String, +} + +#[derive(thiserror::Error, Debug)] +pub enum CreateDyrpcError { + #[error("Failed to deploy dyRPC: {0}")] + FailedToDeployContract(String, String), + + #[error("dyRPC response is not json: {0}")] + ResponseNotJson(reqwest::Error), + + #[error("dyRPC api failed: {0}")] + ApiFailed(reqwest::Error), +} + +pub async fn create_dyrpc_api_key() -> Result { + let client = Client::new(); + let response = client + .post("https://api.dyrpc.network/generate") + .send() + .await + .map_err(CreateDyrpcError::ApiFailed)?; + + if response.status().is_success() { + let api_key = response.text().await.map_err(CreateDyrpcError::ResponseNotJson)?; + Ok(api_key) + } else { + Err(CreateDyrpcError::FailedToDeployContract( + response.status().to_string(), + response.text().await.unwrap_or_default(), + )) + } +} + +pub async fn deploy_dyrpc_contract( + api_key: &str, + clone_meta: &CloneMeta, + compiled_contract: &CompiledContract, +) -> Result> { + let result = deploy_contract( + &clone_meta.address, + api_key, + &compiled_contract.bytecode.object, + &clone_meta.constructor_arguments, + ) + .await?; + + let re = Regex::new(r"/eth/([a-fA-F0-9]{64})/").unwrap(); + let rpc_url = re + .replace(&result.rpc_url, "/eth/{RINDEXER_PHANTOM_API_KEY}/") + .to_string() + .replace("{RINDEXER_PHANTOM_API_KEY}", "${RINDEXER_PHANTOM_API_KEY}"); + + Ok(rpc_url) +} + +async fn deploy_contract( + address: &str, + api_key: &str, + new_bytecode: &str, + constructor_args_bytecode: &str, +) -> Result { + let url = format!("https://node.dyrpc.network/eth/{}/overlay/put", api_key); + + let mut overlays = std::collections::HashMap::new(); + overlays.insert( + address, + DeployDyrpcDetails { + creation_code: new_bytecode, + constructor_args: constructor_args_bytecode, + }, + ); + + let request_body = DeployDyrpcRequest { overlays }; + + let client = Client::new(); + let response = client + .put(&url) + .header("Content-Type", "application/json") + .json(&request_body) + .send() + .await + .map_err(CreateDyrpcError::ApiFailed)?; + + if response.status().is_success() { + let overlay_response: DeployDyrpcContractResponse = + response.json().await.map_err(CreateDyrpcError::ResponseNotJson)?; + Ok(overlay_response) + } else { + Err(CreateDyrpcError::FailedToDeployContract( + response.status().to_string(), + response.text().await.unwrap_or_default(), + )) + } +} diff --git a/core/src/phantom/mod.rs b/core/src/phantom/mod.rs new file mode 100644 index 00000000..480aa1fd --- /dev/null +++ b/core/src/phantom/mod.rs @@ -0,0 +1,5 @@ +pub mod common; +mod dyrpc; +pub mod shadow; + +pub use dyrpc::{create_dyrpc_api_key, deploy_dyrpc_contract, CreateDyrpcError}; diff --git a/core/src/phantom/shadow.rs b/core/src/phantom/shadow.rs new file mode 100644 index 00000000..23fc45ef --- /dev/null +++ b/core/src/phantom/shadow.rs @@ -0,0 +1,202 @@ +use std::{collections::BTreeMap, path::Path, process::Command}; + +use ethers_solc::{ + artifacts::{Contract, Contracts, Error, SourceFile}, + CompilerOutput, +}; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{manifest::phantom::PhantomShadow, phantom::common::CloneMeta}; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +pub struct ShadowSourceFile { + id: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +pub struct ShadowCompilerOutput { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub errors: Vec, + + #[serde(default)] + pub sources: BTreeMap, + + #[serde(default)] + pub contracts: Contracts, +} + +impl ShadowCompilerOutput { + pub fn from_compile_output(output: CompilerOutput) -> Self { + let sources = output + .sources + .into_iter() + .map(|(key, value)| { + let new_key = key.split("lib/").last().unwrap_or_default().to_string(); + let new_value = ShadowSourceFile { id: value.id }; + (new_key, new_value) + }) + .collect(); + + let contracts = output + .contracts + .into_iter() + .map(|(file, contracts)| { + let new_file = file.split("lib/").last().unwrap_or_default().to_string(); + let new_contracts = contracts.into_iter().collect(); + (new_file, new_contracts) + }) + .collect(); + + ShadowCompilerOutput { errors: output.errors, sources, contracts } + } +} + +#[derive(thiserror::Error, Debug)] +pub enum DeployShadowError { + #[error("Could not run forge build")] + CouldNotCompileContract, + + #[error("Failed to read format json from forge build")] + CouldNotReadFormatJson, + + #[error("Invalid compiler output from format json")] + InvalidCompilerOutputFromFormatJson, + + #[error("Failed to deploy contract: {0}")] + FailedToDeployContract(String, String), + + #[error("dyRPC response is not json: {0}")] + ResponseNotJson(reqwest::Error), + + #[error("dyRPC api failed: {0}")] + ApiFailed(reqwest::Error), +} + +pub async fn deploy_shadow_contract( + api_key: &str, + deploy_in: &Path, + clone_meta: &CloneMeta, + shadow_details: &PhantomShadow, +) -> Result { + let output = Command::new("forge") + .arg("build") + .arg("--format-json") + .arg("--force") + .current_dir(deploy_in) + .output() + .map_err(|_| DeployShadowError::CouldNotCompileContract)?; + + if output.status.success() { + let stdout_str = std::str::from_utf8(&output.stdout) + .map_err(|_| DeployShadowError::CouldNotReadFormatJson)?; + + let compiler_output: CompilerOutput = forge_to_solc(stdout_str) + .map_err(|_| DeployShadowError::InvalidCompilerOutputFromFormatJson)?; + + let shadow_compiler_output = ShadowCompilerOutput::from_compile_output(compiler_output); + + deploy_shadow(api_key, clone_meta, shadow_details, shadow_compiler_output).await + } else { + Err(DeployShadowError::CouldNotReadFormatJson) + } +} + +fn forge_to_solc(stdout_str: &str) -> Result { + let val: Value = serde_json::from_str(stdout_str)?; + let errors_arr = val["errors"].as_array().unwrap(); + let contract_objs_val = val["contracts"].as_object().unwrap(); + let sources_obj_val = val["sources"].as_object().unwrap(); + + let mut contracts = Contracts::new(); + let mut sources: BTreeMap = BTreeMap::new(); + + let errors = errors_arr + .iter() + .map(|e| serde_json::from_value::(e.clone()).unwrap()) + .collect::>(); + + for (file, value) in contract_objs_val.into_iter() { + let obj = value.as_object().unwrap(); + let modules = obj.into_iter().collect::>(); + let mut contracts_map: BTreeMap = BTreeMap::new(); + for (module_name, contract_objs_wrapper) in modules { + // Note: Forge output has an array at this level, but solc output + // seems to be an object. Is there a guarantee to only be one object? + let contract = &contract_objs_wrapper[0]["contract"]; + let parsed_contract: Contract = serde_json::from_value(contract.clone())?; + + contracts_map.insert(module_name.clone(), parsed_contract); + } + contracts.insert(file.clone(), contracts_map); + } + + for (file, value) in sources_obj_val.into_iter() { + let arr = value.as_array().unwrap(); + // Note: Forge output has an array at this level, but solc output + // seems to be an object. Is there a guarantee to only be one object? + let source_file: SourceFile = serde_json::from_value(arr[0]["source_file"].clone())?; + sources.insert(file.clone(), source_file); + } + + Ok(CompilerOutput { errors, sources, contracts }) +} + +#[derive(Serialize, Deserialize, Debug)] +struct ShadowBodyContract { + address: String, + + #[serde(rename = "compilerOutput")] + compiler_output: ShadowCompilerOutput, +} + +#[derive(Serialize, Deserialize, Debug)] +struct DeployShadowBody { + #[serde(rename = "shadowedContracts")] + shadowed_contracts: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct DeployShadowResponse { + pub fork_id: String, + pub fork_version: u64, + pub rpc_url: String, +} + +async fn deploy_shadow( + api_key: &str, + clone_meta: &CloneMeta, + shadow_details: &PhantomShadow, + shadow_compiler_output: ShadowCompilerOutput, +) -> Result { + let client = Client::new(); + + let response = client + // https://api.staging.shadow.xyz + // https://api.shadow.xyz + .post(format!("https://api.shadow.xyz/v1/{}/deploy", shadow_details.fork_id)) + .header("X-SHADOW-API-KEY", api_key) + .json(&DeployShadowBody { + shadowed_contracts: vec![ShadowBodyContract { + address: clone_meta.address.clone(), + compiler_output: shadow_compiler_output, + }], + }) + .send() + .await + .map_err(DeployShadowError::ApiFailed)?; + + if response.status().is_success() { + let response: DeployShadowResponse = + response.json().await.map_err(DeployShadowError::ResponseNotJson)?; + + Ok(response.rpc_url) + } else { + Err(DeployShadowError::FailedToDeployContract( + response.status().to_string(), + response.text().await.unwrap_or_default(), + )) + } +} diff --git a/core/src/provider.rs b/core/src/provider.rs index 2c53ebe2..4fc60acf 100644 --- a/core/src/provider.rs +++ b/core/src/provider.rs @@ -9,6 +9,7 @@ use ethers::{ providers::{Http, Provider, ProviderError, RetryClient, RetryClientBuilder}, types::{Block, BlockNumber, H256, U256, U64}, }; +use reqwest::header::HeaderMap; use thiserror::Error; use tokio::sync::Mutex; use url::Url; @@ -73,17 +74,23 @@ impl JsonRpcCachedProvider { pub enum RetryClientError { #[error("http provider can't be created for {0}: {1}")] HttpProviderCantBeCreated(String, String), + + #[error("Could not build client: {0}")] + CouldNotBuildClient(#[from] reqwest::Error), } pub fn create_client( rpc_url: &str, compute_units_per_second: Option, max_block_range: Option, + custom_headers: HeaderMap, ) -> Result, RetryClientError> { let url = Url::parse(rpc_url).map_err(|e| { RetryClientError::HttpProviderCantBeCreated(rpc_url.to_string(), e.to_string()) })?; - let provider = Http::new(url); + let client = reqwest::Client::builder().default_headers(custom_headers).build()?; + + let provider = Http::new_with_client(url, client); let instance = Provider::new( RetryClientBuilder::default() // assume minimum compute units per second if not provided as growth plan standard @@ -117,6 +124,7 @@ impl CreateNetworkProvider { &network.rpc, network.compute_units_per_second, network.max_block_range, + manifest.get_custom_headers(), )?; result.push(CreateNetworkProvider { network_name: network.name.clone(), @@ -135,14 +143,14 @@ mod tests { #[test] fn test_create_retry_client() { let rpc_url = "http://localhost:8545"; - let result = create_client(rpc_url, Some(660), None); + let result = create_client(rpc_url, Some(660), None, HeaderMap::new()); assert!(result.is_ok()); } #[test] fn test_create_retry_client_invalid_url() { let rpc_url = "invalid_url"; - let result = create_client(rpc_url, Some(660), None); + let result = create_client(rpc_url, Some(660), None, HeaderMap::new()); assert!(result.is_err()); if let Err(RetryClientError::HttpProviderCantBeCreated(url, _)) = result { assert_eq!(url, rpc_url); diff --git a/documentation/docs/pages/docs/changelog.mdx b/documentation/docs/pages/docs/changelog.mdx index b9789bcb..b7b43187 100644 --- a/documentation/docs/pages/docs/changelog.mdx +++ b/documentation/docs/pages/docs/changelog.mdx @@ -6,9 +6,14 @@ ### Features ------------------------------------------------- +- feat: support for phantom events - https://rindexer.xyz/docs/start-building/phantom + ### Bug fixes ------------------------------------------------- +fix: resolve issue with no inputs in events syntax error for postgres +fix: better error message when etherscan is not supported for network + ### Breaking changes ------------------------------------------------- diff --git a/documentation/docs/pages/docs/introduction/installation.mdx b/documentation/docs/pages/docs/introduction/installation.mdx index 458ea860..30c2ac13 100644 --- a/documentation/docs/pages/docs/introduction/installation.mdx +++ b/documentation/docs/pages/docs/introduction/installation.mdx @@ -34,6 +34,7 @@ Commands: add Add elements such as contracts to the rindexer.yaml file codegen Generates rust code based on rindexer.yaml or graphql queries delete Delete data from the postgres database or csv files + phantom Use phantom events to add your own events to contracts help Print this message or the help of the given subcommand(s) Options: diff --git a/documentation/docs/pages/docs/introduction/other-indexing-tools.mdx b/documentation/docs/pages/docs/introduction/other-indexing-tools.mdx index 01868462..ff2440e2 100644 --- a/documentation/docs/pages/docs/introduction/other-indexing-tools.mdx +++ b/documentation/docs/pages/docs/introduction/other-indexing-tools.mdx @@ -23,10 +23,22 @@ If you want decentralised provable indexing `TheGraph` is the tool you should be ## Shadow - Paid -`Shadow` allows you to add custom events to smart contracts and is a very powerful indexing service. -I can see a future where rindexer allows you to modify contracts and then resync them which points to shadow events with no-code. +:::note +rindexer has first-party support for phantom events powered by Shadow, you can read more about it [here](/docs/start-building/phantom). +::: + +`Shadow` allows you to add custom events and view functions to smart contracts and is a very powerful indexing service. The `Shadow` team are doing great work and I have a lot of respect for them and the work they are doing. +## dyRPC - Paid + +:::note +rindexer has first-party support for phantom events powered by dyRPC, you can read more about it [here](/docs/start-building/phantom). +::: + +`dyRPC` is a tool built on top of overlay which can be ran on any erigon node and allows you to also modify the contract's source code +adding gasless custom events and view functions. The `dyRPC` team are awesome. + ## Cryo - Free `Cryo` is a great way to extract data from all EVM chains and is awesome for data analysis and research. It also is powered diff --git a/documentation/docs/pages/docs/introduction/what-is-rindexer.mdx b/documentation/docs/pages/docs/introduction/what-is-rindexer.mdx index 04de6a53..2e731e13 100644 --- a/documentation/docs/pages/docs/introduction/what-is-rindexer.mdx +++ b/documentation/docs/pages/docs/introduction/what-is-rindexer.mdx @@ -1,7 +1,7 @@ # What is rindexer ? :::info -Note rindexer is brand new and only in beta and actively under development, things will change and bugs will exist - if you find any bugs or have any +Note rindexer is brand new and actively under development, things will change and bugs will exist - if you find any bugs or have any feature requests please open an issue on [github](https://github.com/joshstevens19/rindexer/issues). ::: diff --git a/documentation/docs/pages/docs/references/cli.mdx b/documentation/docs/pages/docs/references/cli.mdx index 18f3713a..a0c05afc 100644 --- a/documentation/docs/pages/docs/references/cli.mdx +++ b/documentation/docs/pages/docs/references/cli.mdx @@ -11,6 +11,7 @@ Commands: add Add elements such as contracts to the rindexer.yaml file codegen Generates rust code based on rindexer.yaml or graphql queries delete Delete data from the postgres database or csv files + phantom Use phantom events to add your own events to contracts help Print this message or the help of the given subcommand(s) Options: @@ -114,3 +115,25 @@ It will ask you questions in the terminal to determine what you want to delete. ```bash Usage: rindexer delete ``` + +## phantom + +```bash +Example: `rindexer phantom init` or `rindexer phantom clone --contract-name --network ` or `rindexer phantom compile --contract-name --network ` or `rindexer phantom deploy --contract-name --network ` + +Usage: rindexer phantom [OPTIONS] + +Commands: + init Sets up phantom events on rindexer + clone Clone the contract with the network you wish to add phantom events to + compile Compiles the phantom contract + deploy Deploy the modified phantom contract + help Print this message or the help of the given subcommand(s) + +Options: + -p, --path + optional - The path to create the project in, default will be where the command is run + + -h, --help + Print help (see a summary with '-h') +``` diff --git a/documentation/docs/pages/docs/start-building/add.mdx b/documentation/docs/pages/docs/start-building/add.mdx index 8f1ad9a7..6fb2c22a 100644 --- a/documentation/docs/pages/docs/start-building/add.mdx +++ b/documentation/docs/pages/docs/start-building/add.mdx @@ -9,8 +9,8 @@ You must have networks setup in the YAML to be able to use this command. You can ::: This allows you download the contracts metadata alongside the ABI from an supported network using Etherscan APIs. -This uses no API keys to try to download the ABIs, this means you will get rate limited if you use this too much -or if many people use this at the same time. +This uses a shared Etherscan API key to try to download the ABIs, this means you will get rate limited if you use this too much +or if many people use this at the same time. You can add your own API key [here](/docs/start-building/yaml-config/global#etherscan_api_key). You can see all the chains you can download ABIs from [here](https://docs.etherscan.io/contract-verification/supported-chains). diff --git a/documentation/docs/pages/docs/start-building/create-new-project.mdx b/documentation/docs/pages/docs/start-building/create-new-project.mdx index 073e4c44..41a1d725 100644 --- a/documentation/docs/pages/docs/start-building/create-new-project.mdx +++ b/documentation/docs/pages/docs/start-building/create-new-project.mdx @@ -114,7 +114,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json diff --git a/documentation/docs/pages/docs/start-building/live-indexing-and-historic.mdx b/documentation/docs/pages/docs/start-building/live-indexing-and-historic.mdx index 3020a121..93239ec2 100644 --- a/documentation/docs/pages/docs/start-building/live-indexing-and-historic.mdx +++ b/documentation/docs/pages/docs/start-building/live-indexing-and-historic.mdx @@ -26,7 +26,7 @@ contracts: - name: RocketPoolETH // [!code focus] details: // [!code focus] - network: ethereum // [!code focus] - address: 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] start_block: 18600000 // [!code focus] end_block: 18718056 // [!code focus] abi: ./abis/RocketTokenRETH.abi.json @@ -61,7 +61,7 @@ contracts: - name: RocketPoolETH // [!code focus] details: // [!code focus] - network: ethereum // [!code focus] - address: 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] abi: ./abis/RocketTokenRETH.abi.json include_events: - Transfer @@ -96,7 +96,7 @@ contracts: - name: RocketPoolETH // [!code focus] details: // [!code focus] - network: ethereum // [!code focus] - address: 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] start_block: 18600000 // [!code focus] abi: ./abis/RocketTokenRETH.abi.json include_events: diff --git a/documentation/docs/pages/docs/start-building/phantom.mdx b/documentation/docs/pages/docs/start-building/phantom.mdx new file mode 100644 index 00000000..e2b74539 --- /dev/null +++ b/documentation/docs/pages/docs/start-building/phantom.mdx @@ -0,0 +1,374 @@ +# Phantom events + +Phantom events enable you to add custom events or modify existing events for any smart contract. +This feature allows you to extract any data you need from smart contracts without the constraints of on-chain modifications. + +## What are Phantom Events? +Phantom events are gasless events logged in an off-chain execution environment that mirrors the mainnet state in real-time. +They provide a solution for obtaining additional information from a contract without incurring extra gas costs for users. + +## Why Use Phantom Events? + +Whether you don't control the contract or want to avoid additional gas costs, phantom events offer a flexible solution +for diverse data needs and use cases. Powered by [Shadow](https://www.shadow.xyz/) and [dyRPC](https://ui.dyrpc.network/), +they are part of the rindexer suite designed to simplify being able to use these powerful features. + +## Getting Started with Phantom Events +rindexer abstracts away the complexity and offers first-party support for implementing phantom events. +It utilizes Etherscan APIs to download source code and ABIs for the contracts you want to index. +Note that the shared Etherscan API key may lead to rate limits if heavily used. +To avoid this, we recommend adding your own API key here [here](/docs/start-building/yaml-config/global#etherscan_api_key). + +Right let's get started with phantom events. + +## Providers + +:::warning +To use phantom events you will need to have a provider, rindexer just offers first-party support for implementing phantom events. +::: + +### [Shadow](https://www.shadow.xyz/) + +Shadow enables you to modify a deployed contract's source code to add gasless custom event logs and view functions on a shadow fork that +is instrumented to mirror mainnet state in realtime. + +#### Networks Supported + +- Ethereum + +### [dyRPC](https://ui.dyrpc.network/) + +dyRPC is a tool built on top of overlay which can be ran on any erigon node and allows you to also modify the contract's source code +adding gasless custom event logs and view functions. + +#### Networks Supported + +- Ethereum + +## Dependencies + +### Installing Foundry + +:::info +If you do not have `foundry` installed it will install it for you when you run the `init` command but we recommend you install it yourself. +::: + +foundry is required to be installed to compile the contracts. + +```bash +curl -L https://foundry.paradigm.xyz | bash +``` + +if you already have got foundry installed you can run `foundryup` to update it. + +## Init + +rindexer uses its CLI first approach for everything and phantom events behaves the same way. Each rindexer project by default +does not have phantom events enabled you have to set them up for each project. + +To enable phantom events for your rindexer project you can run the following command: + +```bash +rindexer phantom init +``` + +### Required information + +- Shadow + - API key (generate on the shadow portal) + - Fork ID (generate on the shadow portal) +- dyRPC + - API key (generate on the dyRPC portal or use "new" to generate a new one) + +You will be asked to pick your provider and add your API key. It will save the API key +int the `.env` file under `RINDEXER_PHANTOM_API_KEY` in your project directory. +It will also add your phantom provider to the `rindexer.yaml` file. + +## Clone + +As the `rindexer.yaml` file is defined by you we use these contract names and network names to allow you to +easily understand what you are cloning. + +:::info +Only verified contracts on Etherscan can be cloned, if you wish to use an unverified contract it will still work +but you will have to create the foundry project manually in the `phantom` folder. +::: + +```bash +rindexer phantom clone --contract-name --network +``` + +lets say we had a `rindexer.yaml` file like this: + +```yaml +name: RocketPoolETHIndexer +description: My first rindexer project +repository: https://github.com/joshstevens19/rindexer +project_type: no-code +networks: +- name: ethereum + chain_id: 1 + rpc: https://mainnet.gateway.tenderly.co +storage: + postgres: + enabled: true + drop_each_run: true +contracts: +- name: RocketPoolETH + details: + - network: ethereum + address: "0xae78736cd615f374d3085123a210448e74fc6393" + start_block: '18600000' + end_block: '18718056' + abi: ./abis/RocketTokenRETH.abi.json + include_events: + - Transfer +``` + +to clone this contract you would run the following command + +```bash +rindexer phantom clone RocketPoolETH ethereum +``` + +This will create you a folder called `phantom` in the root of your rindexer project and also create the network name +to make it easier to find contracts you have cloned. for example in the above example it will create a folder called `phantom/ethereum/` +and inside it will have a folder called `RocketPoolETH/` which will have your solidity project files and the contract ABI. +This folder will contain all the phantom contracts you have cloned. + +You can now go to the contracts folder and start making changes to the phantom contracts. + +## Add your own event + +Above we cloned `RocketPoolETH` on `ethereum` lets open up `RocketTokenRETH.sol` and add a phantom event on transfer hook. + +```solidity +contract RocketTokenRETH is RocketBase, ERC20, RocketTokenRETHInterface { + using SafeMath for uint; + + event EtherDeposited(address indexed from, uint256 amount, uint256 time); + event TokensMinted(address indexed to, uint256 amount, uint256 ethAmount, uint256 time); + event TokensBurned(address indexed from, uint256 amount, uint256 ethAmount, uint256 time); + event PhantomTransferTime(address indexed from, uint256 time); // [!code focus] + + ... + + function _beforeTokenTransfer(address from, address, uint256) internal override { + // emit your own event + emit PhantomTransferTime(from, block.timestamp); // [!code focus] + + // Don't run check if this is a mint transaction + if (from != address(0)) { + // Check which block the user's last deposit was + bytes32 key = keccak256(abi.encodePacked("user.deposit.block", from)); + uint256 lastDepositBlock = getUint(key); + if (lastDepositBlock > 0) { + // Ensure enough blocks have passed + uint256 depositDelay = getUint(keccak256(abi.encodePacked(keccak256("dao.protocol.setting.network"), "network.reth.deposit.delay"))); + uint256 blocksPassed = block.number.sub(lastDepositBlock); + require(blocksPassed > depositDelay, "Not enough time has passed since deposit"); + // Clear the state as it's no longer necessary to check this until another deposit is made + deleteUint(key); + } + } + } +``` + +That is it you can now compile and deploy your phantom contract which we will go over in the next section. + +## Editing existing events + +You can edit any event to whatever you want for example lets say we wanted to change `TokensMinted` to include +the new balance of the `to` address after minted. + +```solidity +contract RocketTokenRETH is RocketBase, ERC20, RocketTokenRETHInterface { + using SafeMath for uint; + + event EtherDeposited(address indexed from, uint256 amount, uint256 time); + event TokensMinted(uint256 newBalance, address indexed to, uint256 amount, uint256 ethAmount, uint256 time); // [!code focus] + event TokensBurned(address indexed from, uint256 amount, uint256 ethAmount, uint256 time); + event PhantomTransferTime(address indexed from, uint256 time); + + ... + + function mint(uint256 _ethAmount, address _to) override external onlyLatestContract("rocketDepositPool", msg.sender) { + // Get rETH amount + uint256 rethAmount = getRethValue(_ethAmount); + // Check rETH amount + require(rethAmount > 0, "Invalid token mint amount"); + // Update balance & supply + _mint(_to, rethAmount); + // Emit tokens minted event + emit TokensMinted(balanceOf(_to), _to, rethAmount, _ethAmount, block.timestamp); // [!code focus] + } +``` + +That is it you can now compile and deploy your phantom contract which we will go over in the next section. + +:::info +If editing different events on different networks (aka your indexing `RocketPoolETH` on ethereum as well as base) +your contract details should be separate for each network in the `rindexer.yaml` file, +as when you deploy the phantom contract it will remap the new ABI and if your events now do not match the types +it will error. +::: + +## Compile + +:::info +rindexer uses `foundry` to clone and compile the contracts. +::: + +To compile the phantom contracts you can run the following command: + +```bash +rindexer phantom compile --contract-name --network +``` + +So using the same yaml example as above you would run the following command: + +```bash +rindexer phantom compile --contract-name RocketPoolETH --network ethereum +``` + +This will show you the same compile errors as `foundry` would show you if you have made any mistakes. + +## Deploy + +Deploying your phantom contract is different to deploying a normal contract. rindexer will take care of uploading +the new phantom contract to the provider and all the mappings for you in the `rindexer.yaml` file. + +```bash +rindexer phantom deploy --contract-name --network +``` + +So using the same yaml example as above you would run the following command: + +```bash +rindexer phantom deploy --contract-name RocketPoolETH --network ethereum +``` + +This will do a few things to your yaml file: + +1. It will add the phantom network to the `rindexer.yaml` file this is always named `phantom_${NETWORK_NAME}_${CONTRACT_NAME}` + +```yaml +name: RocketPoolETHIndexer +description: My first rindexer project +repository: https://github.com/joshstevens19/rindexer +project_type: no-code +networks: +- name: ethereum + chain_id: 1 + rpc: https://mainnet.gateway.tenderly.co +- name: phantom_ethereum_RocketPoolETH // [!code focus] + chain_id: 1 // [!code focus] + rpc: PROVIDER_RPC // [!code focus] +... +``` + +2. It will change the `contracts` section to point the contract details to the phantom network. + +```yaml +name: RocketPoolETHIndexer +description: My first rindexer project +repository: https://github.com/joshstevens19/rindexer +project_type: no-code +networks: +- name: ethereum + chain_id: 1 + rpc: https://mainnet.gateway.tenderly.co +- name: phantom_ethereum_RocketPoolETH // [!code focus] + chain_id: 1 // [!code focus] + rpc: PROVIDER_RPC // [!code focus] +storage: + postgres: + enabled: true + drop_each_run: true +contracts: +- name: RocketPoolETH + details: + - network: phantom_ethereum_RocketPoolETH // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" + start_block: '18600000' + end_block: '18718056' + abi: ./abis/phantom_ethereum_RocketPoolETH.abi.json + include_events: + - Transfer +... +``` + +3. It will upload the new ABI to your `abi` folder named the same as the network name but with the `.abi.json` extension. + +```yaml +name: RocketPoolETHIndexer +description: My first rindexer project +repository: https://github.com/joshstevens19/rindexer +project_type: no-code +networks: +- name: ethereum + chain_id: 1 + rpc: https://mainnet.gateway.tenderly.co +- name: phantom_ethereum_RocketPoolETH // [!code focus] + chain_id: 1 // [!code focus] + rpc: PROVIDER_RPC // [!code focus] +storage: + postgres: + enabled: true + drop_each_run: true +contracts: +- name: RocketPoolETH + details: + - network: phantom_ethereum_RocketPoolETH // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" + start_block: '18600000' + end_block: '18718056' + abi: ./abis/phantom_ethereum_RocketPoolETH.abi.json // [!code focus] + include_events: + - Transfer +... +``` + +right now lets include the `PhantomTransferTime` event in our yaml file `include_events` array so we can index it. + +```yaml +name: RocketPoolETHIndexer +description: My first rindexer project +repository: https://github.com/joshstevens19/rindexer +project_type: no-code +networks: +- name: ethereum + chain_id: 1 + rpc: https://mainnet.gateway.tenderly.co +- name: phantom_ethereum_RocketPoolETH + chain_id: 1 + rpc: PROVIDER_RPC +storage: + postgres: + enabled: true + drop_each_run: true +contracts: +- name: RocketPoolETH + details: + - network: phantom_ethereum_RocketPoolETH + address: "0xae78736cd615f374d3085123a210448e74fc6393" + start_block: '18600000' + end_block: '18718056' + abi: ./abis/phantom_ethereum_RocketPoolETH.abi.json + include_events: + - PhantomTransferTime // [!code focus] +... +``` + +## Indexing + +everything else is same as before so you can run `rindexer start all` the database tables will all be created, +indexer will start indexing and the GraphQL API will be available at http://localhost:3001/graphql. + +:::info +Expect slower indexing as you will have to wait for the phantom provider to index the events. +All phantom providers at the moment use block ranges over optimal ranges so phantom events will be slower +than normal events. +::: + +That is it you now have created phantom events with rindexer. \ No newline at end of file diff --git a/documentation/docs/pages/docs/start-building/rust-project-deep-dive/indexers.mdx b/documentation/docs/pages/docs/start-building/rust-project-deep-dive/indexers.mdx index e687c641..32627a84 100644 --- a/documentation/docs/pages/docs/start-building/rust-project-deep-dive/indexers.mdx +++ b/documentation/docs/pages/docs/start-building/rust-project-deep-dive/indexers.mdx @@ -44,7 +44,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -438,7 +438,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -503,7 +503,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -611,7 +611,7 @@ contracts: - name: RocketPoolETH // [!code focus] details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -731,7 +731,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -848,7 +848,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json diff --git a/documentation/docs/pages/docs/start-building/yaml-config/contracts.mdx b/documentation/docs/pages/docs/start-building/yaml-config/contracts.mdx index 95916928..fb45083d 100644 --- a/documentation/docs/pages/docs/start-building/yaml-config/contracts.mdx +++ b/documentation/docs/pages/docs/start-building/yaml-config/contracts.mdx @@ -74,7 +74,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] ``` To listen to many contract addresses you can provide an array of addresses. @@ -95,8 +95,8 @@ contracts: // [!code focus] details: - network: ethereum address: - - 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] - - 0x2FD5c1659A82E87217DF254f3D4b71A22aE43eE1 // [!code focus] + - "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] + - "0x2FD5c1659A82E87217DF254f3D4b71A22aE43eE1" // [!code focus] ``` ### filter @@ -230,12 +230,12 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" indexed_filters: // [!code focus] - event_name: Approval // [!code focus] indexed_1: - - 0xd87b8e0db0cf9cbf9963c035a6ad72d614e37fd5 // [!code focus] - - 0x0338ce5020c447f7e668dc2ef778025ce398266b // [!code focus] + - "0xd87b8e0db0cf9cbf9963c035a6ad72d614e37fd5" // [!code focus] + - "0x0338ce5020c447f7e668dc2ef778025ce398266b" // [!code focus] ``` Another example using filters is if you wanted to get all the approvals for any token for owner `0xd87b8e0db0cf9cbf9963c035a6ad72d614e37fd5` @@ -291,7 +291,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 // [!code focus] ``` @@ -317,7 +317,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 // [!code focus] ``` @@ -346,11 +346,11 @@ contracts: // [!code focus] - name: RocketPoolETH details: // [!code focus] - network: ethereum // [!code focus] - address: 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] + address: "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] start_block: 18600000 // [!code focus] end_block: 18718056 // [!code focus] - network: base // [!code focus] - address: 0xba25348cd615f374d3085123a210448e74fa3333 // [!code focus] + address: "0xba25348cd615f374d3085123a210448e74fa3333" // [!code focus] start_block: 18118056 // [!code focus] end_block: 18918056 // [!code focus] ``` @@ -374,7 +374,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json // [!code focus] @@ -403,7 +403,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -439,7 +439,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -493,7 +493,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -541,7 +541,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -587,7 +587,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -621,7 +621,7 @@ contracts: // [!code focus] - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json diff --git a/documentation/docs/pages/docs/start-building/yaml-config/global.mdx b/documentation/docs/pages/docs/start-building/yaml-config/global.mdx index aa9d99d6..83590dc5 100644 --- a/documentation/docs/pages/docs/start-building/yaml-config/global.mdx +++ b/documentation/docs/pages/docs/start-building/yaml-config/global.mdx @@ -27,7 +27,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -67,7 +67,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -103,7 +103,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -137,7 +137,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -172,7 +172,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json @@ -209,7 +209,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18900000 end_block: 19000000 abi: ./abis/RocketTokenRETH.abi.json diff --git a/documentation/docs/pages/docs/start-building/yaml-config/graphql.mdx b/documentation/docs/pages/docs/start-building/yaml-config/graphql.mdx index 872c1e59..eb0accd9 100644 --- a/documentation/docs/pages/docs/start-building/yaml-config/graphql.mdx +++ b/documentation/docs/pages/docs/start-building/yaml-config/graphql.mdx @@ -26,7 +26,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -57,7 +57,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -91,7 +91,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json diff --git a/documentation/docs/pages/docs/start-building/yaml-config/index.mdx b/documentation/docs/pages/docs/start-building/yaml-config/index.mdx index b42f053a..200e125c 100644 --- a/documentation/docs/pages/docs/start-building/yaml-config/index.mdx +++ b/documentation/docs/pages/docs/start-building/yaml-config/index.mdx @@ -44,7 +44,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -77,7 +77,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -106,7 +106,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" abi: ./abis/RocketTokenRETH.abi.json include_events: - Transfer @@ -133,7 +133,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: 18600000 abi: ./abis/RocketTokenRETH.abi.json include_events: @@ -164,8 +164,8 @@ contracts: details: - network: ethereum address: - - 0xae78736cd615f374d3085123a210448e74fc6393 // [!code focus] - - 0x2FD5c1659A82E87217DF254f3D4b71A22aE43eE1 // [!code focus] + - "0xae78736cd615f374d3085123a210448e74fc6393" // [!code focus] + - "0x2FD5c1659A82E87217DF254f3D4b71A22aE43eE1" // [!code focus] start_block: 18600000 end_block: 18718056 abi: ./abis/RocketTokenRETH.abi.json @@ -195,7 +195,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" indexed_filters: // [!code focus] - event_name: Transfer // [!code focus] indexed_1: // [!code focus] diff --git a/documentation/vocs.config.ts b/documentation/vocs.config.ts index 9e3aa6bb..bc1881b7 100644 --- a/documentation/vocs.config.ts +++ b/documentation/vocs.config.ts @@ -70,6 +70,10 @@ export default defineConfig({ text: 'Delete', link: '/docs/start-building/delete', }, + { + text: 'Phantom Events', + link: '/docs/start-building/phantom', + }, { text: 'Rust Project Deep Dive', link: '/docs/start-building/rust-project-deep-dive', diff --git a/examples/rindexer_demo_cli/.env b/examples/rindexer_demo_cli/.env new file mode 100644 index 00000000..c8a2c308 --- /dev/null +++ b/examples/rindexer_demo_cli/.env @@ -0,0 +1,2 @@ +DATABASE_URL=postgresql://postgres:rindexer@localhost:5440/postgres +POSTGRES_PASSWORD=rindexer diff --git a/examples/rindexer_demo_cli/abis/RocketTokenRETH.abi.json b/examples/rindexer_demo_cli/abis/RocketTokenRETH.abi.json new file mode 100644 index 00000000..12c6306a --- /dev/null +++ b/examples/rindexer_demo_cli/abis/RocketTokenRETH.abi.json @@ -0,0 +1,537 @@ +[ + { + "inputs":[ + { + "internalType":"contract RocketStorageInterface", + "name":"_rocketStorageAddress", + "type":"address" + } + ], + "stateMutability":"nonpayable", + "type":"constructor" + }, + { + "anonymous":false, + "inputs":[ + { + "indexed":true, + "internalType":"address", + "name":"owner", + "type":"address" + }, + { + "indexed":true, + "internalType":"address", + "name":"spender", + "type":"address" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"value", + "type":"uint256" + } + ], + "name":"Approval", + "type":"event" + }, + { + "anonymous":false, + "inputs":[ + { + "indexed":true, + "internalType":"address", + "name":"from", + "type":"address" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"amount", + "type":"uint256" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"time", + "type":"uint256" + } + ], + "name":"EtherDeposited", + "type":"event" + }, + { + "anonymous":false, + "inputs":[ + { + "indexed":true, + "internalType":"address", + "name":"from", + "type":"address" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"amount", + "type":"uint256" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"ethAmount", + "type":"uint256" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"time", + "type":"uint256" + } + ], + "name":"TokensBurned", + "type":"event" + }, + { + "anonymous":false, + "inputs":[ + { + "indexed":true, + "internalType":"address", + "name":"to", + "type":"address" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"amount", + "type":"uint256" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"ethAmount", + "type":"uint256" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"time", + "type":"uint256" + } + ], + "name":"TokensMinted", + "type":"event" + }, + { + "anonymous":false, + "inputs":[ + { + "indexed":true, + "internalType":"address", + "name":"from", + "type":"address" + }, + { + "indexed":true, + "internalType":"address", + "name":"to", + "type":"address" + }, + { + "indexed":false, + "internalType":"uint256", + "name":"value", + "type":"uint256" + } + ], + "name":"Transfer", + "type":"event" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"owner", + "type":"address" + }, + { + "internalType":"address", + "name":"spender", + "type":"address" + } + ], + "name":"allowance", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"spender", + "type":"address" + }, + { + "internalType":"uint256", + "name":"amount", + "type":"uint256" + } + ], + "name":"approve", + "outputs":[ + { + "internalType":"bool", + "name":"", + "type":"bool" + } + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"account", + "type":"address" + } + ], + "name":"balanceOf", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"uint256", + "name":"_rethAmount", + "type":"uint256" + } + ], + "name":"burn", + "outputs":[ + + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"decimals", + "outputs":[ + { + "internalType":"uint8", + "name":"", + "type":"uint8" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"spender", + "type":"address" + }, + { + "internalType":"uint256", + "name":"subtractedValue", + "type":"uint256" + } + ], + "name":"decreaseAllowance", + "outputs":[ + { + "internalType":"bool", + "name":"", + "type":"bool" + } + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"depositExcess", + "outputs":[ + + ], + "stateMutability":"payable", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"depositExcessCollateral", + "outputs":[ + + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"getCollateralRate", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"uint256", + "name":"_rethAmount", + "type":"uint256" + } + ], + "name":"getEthValue", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"getExchangeRate", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"uint256", + "name":"_ethAmount", + "type":"uint256" + } + ], + "name":"getRethValue", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"getTotalCollateral", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"spender", + "type":"address" + }, + { + "internalType":"uint256", + "name":"addedValue", + "type":"uint256" + } + ], + "name":"increaseAllowance", + "outputs":[ + { + "internalType":"bool", + "name":"", + "type":"bool" + } + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"uint256", + "name":"_ethAmount", + "type":"uint256" + }, + { + "internalType":"address", + "name":"_to", + "type":"address" + } + ], + "name":"mint", + "outputs":[ + + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"name", + "outputs":[ + { + "internalType":"string", + "name":"", + "type":"string" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"symbol", + "outputs":[ + { + "internalType":"string", + "name":"", + "type":"string" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"totalSupply", + "outputs":[ + { + "internalType":"uint256", + "name":"", + "type":"uint256" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"recipient", + "type":"address" + }, + { + "internalType":"uint256", + "name":"amount", + "type":"uint256" + } + ], + "name":"transfer", + "outputs":[ + { + "internalType":"bool", + "name":"", + "type":"bool" + } + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + { + "internalType":"address", + "name":"sender", + "type":"address" + }, + { + "internalType":"address", + "name":"recipient", + "type":"address" + }, + { + "internalType":"uint256", + "name":"amount", + "type":"uint256" + } + ], + "name":"transferFrom", + "outputs":[ + { + "internalType":"bool", + "name":"", + "type":"bool" + } + ], + "stateMutability":"nonpayable", + "type":"function" + }, + { + "inputs":[ + + ], + "name":"version", + "outputs":[ + { + "internalType":"uint8", + "name":"", + "type":"uint8" + } + ], + "stateMutability":"view", + "type":"function" + }, + { + "stateMutability":"payable", + "type":"receive" + } +] \ No newline at end of file diff --git a/examples/rindexer_demo_cli/docker-compose.yml b/examples/rindexer_demo_cli/docker-compose.yml new file mode 100644 index 00000000..f22494b5 --- /dev/null +++ b/examples/rindexer_demo_cli/docker-compose.yml @@ -0,0 +1,15 @@ +volumes: + postgres_data: + driver: local + +services: + postgresql: + image: postgres:16 + shm_size: 1g + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - 5440:5432 + env_file: + - ./.env \ No newline at end of file diff --git a/examples/rindexer_demo_cli/rindexer.yaml b/examples/rindexer_demo_cli/rindexer.yaml new file mode 100644 index 00000000..5518b75c --- /dev/null +++ b/examples/rindexer_demo_cli/rindexer.yaml @@ -0,0 +1,22 @@ +name: RocketPoolETHIndexer +description: My first rindexer project +repository: https://github.com/joshstevens19/rindexer +project_type: no-code +networks: +- name: ethereum + chain_id: 1 + rpc: https://mainnet.gateway.tenderly.co +storage: + postgres: + enabled: true + drop_each_run: true +contracts: +- name: RocketPoolETH + details: + - network: ethereum + address: 0xae78736cd615f374d3085123a210448e74fc6393 + start_block: '18600000' + end_block: '18718056' + abi: ./abis/RocketTokenRETH.abi.json + include_events: + - Transfer diff --git a/rindexer_rust_playground/rindexer.yaml b/rindexer_rust_playground/rindexer.yaml index 0189fc75..e728f15b 100644 --- a/rindexer_rust_playground/rindexer.yaml +++ b/rindexer_rust_playground/rindexer.yaml @@ -18,7 +18,7 @@ contracts: - name: RocketPoolETH details: - network: ethereum - address: 0xae78736cd615f374d3085123a210448e74fc6393 + address: "0xae78736cd615f374d3085123a210448e74fc6393" start_block: '18900000' end_block: '19000000' abi: ./abis/erc20-abi.json diff --git a/rindexer_rust_playground/src/rindexer_lib/typings/networks.rs b/rindexer_rust_playground/src/rindexer_lib/typings/networks.rs index af3f5b49..926eddba 100644 --- a/rindexer_rust_playground/src/rindexer_lib/typings/networks.rs +++ b/rindexer_rust_playground/src/rindexer_lib/typings/networks.rs @@ -8,23 +8,39 @@ use ethers::providers::{Http, Provider, RetryClient}; use ethers::types::U64; use rindexer::{ lazy_static, - provider::{create_client, JsonRpcCachedProvider}, - public_read_env_value, + provider::{create_client, JsonRpcCachedProvider, RetryClientError}, + public_read_env_value, HeaderMap, }; +#[allow(dead_code)] +fn create_shadow_client( + rpc_url: &str, + compute_units_per_second: Option, + max_block_range: Option, +) -> Result, RetryClientError> { + let mut header = HeaderMap::new(); + header.insert( + "X-SHADOW-API-KEY", + public_read_env_value("RINDEXER_PHANTOM_API_KEY").unwrap().parse().unwrap(), + ); + create_client(rpc_url, compute_units_per_second, max_block_range, header) +} + lazy_static! { static ref ETHEREUM_PROVIDER: Arc = create_client( &public_read_env_value("https://mainnet.gateway.tenderly.co") .unwrap_or("https://mainnet.gateway.tenderly.co".to_string()), None, - None + None, + HeaderMap::new(), ) .expect("Error creating provider"); static ref YOMINET_PROVIDER: Arc = create_client( &public_read_env_value("https://yominet.rpc.caldera.xyz/http") .unwrap_or("https://yominet.rpc.caldera.xyz/http".to_string()), None, - Some(U64::from(10000)) + Some(U64::from(10000)), + HeaderMap::new(), ) .expect("Error creating provider"); }