diff --git a/.github/actions/ts-e2e/action.yml b/.github/actions/ts-e2e/action.yml index 962c75b8ba234..248180eb31e98 100644 --- a/.github/actions/ts-e2e/action.yml +++ b/.github/actions/ts-e2e/action.yml @@ -29,7 +29,11 @@ runs: - name: cargo build if: env.s3_file_exist == '' # if empty, we have not built and uploaded this binary to s3 yet run: | - cargo build --bin sui --features indexer + if [[ "${{ inputs.ref }}" == 'devnet' || "${{ inputs.ref }}" == 'testnet' ]]; then + cargo build --bin sui --features indexer + else + cargo build --bin sui + fi shell: bash - name: Dowload from S3 @@ -37,7 +41,7 @@ runs: working-directory: ./target/debug run: | mkdir -p $PWD/target/debug - wget -O target/debug/sui https://sui-releases.s3.us-east-1.amazonaws.com/${{ github.sha }}/debug/sui-pg + wget -O target/debug/sui https://sui-releases.s3.us-east-1.amazonaws.com/${{ github.sha }}/debug/sui chmod +x $PWD/target/debug/sui shell: bash diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d5f02bfa44cc4..a4612d28d78d1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -50,7 +50,7 @@ jobs: - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # pin@v3.0.0 with: version: 9.1.1 - - run: cargo build --bin sui --features indexer --profile dev + - run: cargo build --bin sui --profile dev - name: Install Nodejs uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # pin@v4.0.2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42b6c478c91d5..cec207ffbfac4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,7 +161,6 @@ jobs: if: ${{ env.s3_archive_exist == '' }} shell: bash run: | - [ -f ~/.cargo/env ] && source ~/.cargo/env ; cargo build --bin sui --release --features indexer && mv target/release/sui target/release/sui-pg [ -f ~/.cargo/env ] && source ~/.cargo/env ; cargo build --release && cargo build --profile=dev --bin sui - name: Rename binaries for ${{ matrix.os }} @@ -176,10 +175,6 @@ jobs: mv ./target/release/${binary}${{ env.extention }} ${{ env.TMP_BUILD_DIR }}/${binary}${{ env.extention }} done - # sui-pg is a special binary that is built with the indexer feature for sui start cmd - export binary=$(echo "sui-pg" | tr -d $'\r') - mv ./target/release/${binary}${{ env.extention }} ${{ env.TMP_BUILD_DIR }}/${binary}${{ env.extention }} - mv ./target/debug/sui${{ env.extention }} ${{ env.TMP_BUILD_DIR }}/sui-debug${{ env.extention }} tar -cvzf ./tmp/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz -C ${{ env.TMP_BUILD_DIR }} . [[ ${{ env.sui_tag }} == *"testnet"* ]] && aws s3 cp ./tmp/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz s3://sui-releases/releases/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz || true diff --git a/crates/sui-graphql-rpc/README.md b/crates/sui-graphql-rpc/README.md index faae0637948ed..9c50c1ed20b02 100644 --- a/crates/sui-graphql-rpc/README.md +++ b/crates/sui-graphql-rpc/README.md @@ -68,7 +68,7 @@ For local dev, it might be useful to spin up an indexer as well. Instructions ar ## Compatibility with json-rpc -`cargo run --bin sui --features indexer -- start --with-faucet --force-regenesis --with-indexer --pg-port 5432 --pg-db-name sui_indexer_v2 --with-graphql` +`cargo run --bin sui -- start --with-faucet --force-regenesis --with-indexer --pg-port 5432 --pg-db-name sui_indexer_v2 --with-graphql` `pnpm --filter @mysten/graphql-transport test:e2e` diff --git a/crates/sui-indexer/README.md b/crates/sui-indexer/README.md index d81a7f2e89910..dd57acf635aa1 100644 --- a/crates/sui-indexer/README.md +++ b/crates/sui-indexer/README.md @@ -37,7 +37,7 @@ cargo run --bin sui -- start --with-faucet --force-regenesis If you want to run a local network with the indexer enabled (note that `libpq` is required), you can run the following command after following the steps in the next section to set up an indexer DB: ```sh -cargo run --bin sui --features indexer -- start --with-faucet --force-regenesis --with-indexer --pg-port 5432 --pg-db-name sui_indexer_v2 +cargo run --bin sui -- start --with-faucet --force-regenesis --with-indexer --pg-port 5432 --pg-db-name sui_indexer_v2 ``` ### Running standalone indexer diff --git a/crates/sui-rpc-loadgen/README.md b/crates/sui-rpc-loadgen/README.md index 94ce35e0df541..345ac6342a852 100644 --- a/crates/sui-rpc-loadgen/README.md +++ b/crates/sui-rpc-loadgen/README.md @@ -17,7 +17,7 @@ Run the following command to see available commands: cargo run --bin sui-rpc-loadgen -- -h ``` -To try this locally, refer to the [docs](https://docs.sui.io/guides/developer/getting-started/local-network). Recommend setting `database-url` to an env variable. Note: run `RUST_LOG="consensus=off" cargo run sui --features indexer -- start --with-faucet --force-regenesis --with-indexer` to rebuild. +To try this locally, refer to the [docs](https://docs.sui.io/guides/developer/getting-started/local-network). Recommend setting `database-url` to an env variable. Note: run `RUST_LOG="consensus=off" cargo run sui -- start --with-faucet --force-regenesis --with-indexer` to rebuild. ### Example 1: Get All Checkpoints diff --git a/crates/sui-test-validator/src/main.rs b/crates/sui-test-validator/src/main.rs index 0cf700fdd06bc..b290867ea3ee7 100644 --- a/crates/sui-test-validator/src/main.rs +++ b/crates/sui-test-validator/src/main.rs @@ -7,11 +7,11 @@ fn main() { How to install/build the sui binary IF: A: you only need the basic functionality, so just faucet and no persistence (no indexer, no GraphQL service), build from source as usual (cargo build --bin sui) or download latest archive from release archives (starting from testnet v1.28.1 or devnet v1.29) and use sui binary. B: you need to also start an indexer (--with-indexer ), or a GraphQL service (--with-graphql), you either: - - download latest archive from release archives (starting from testnet v1.28.1 or devnet v1.29) and use sui-pg binary. + - download latest archive from release archives (starting from testnet v1.28.1 or devnet v1.29) and use sui-pg binary (note that with v1.34.0 sui-pg no longer exists in the release. Use `sui` binary instead). OR - - build from source. This requires passing the indexer feature when building the sui binary, as well as having libpq/postgresql dependencies installed (just as when using sui-test-validator): - - cargo build --bin sui --features indexer - - cargo run --features indexer --bin sui -- start --with-faucet --force-regenesis --with-indexer --with-graphql + - build from source. This requires to have libpq/postgresql dependencies installed (just as when using sui-test-validator): + - cargo build --bin sui + - cargo run --bin sui -- start --with-faucet --force-regenesis --with-indexer --with-graphql Running the local network: - (Preferred) In the simplest form, you can replace sui-test-validator with sui start --with-faucet --force-regenesis. This will create a network from a new genesis and start a faucet (127.0.0.1:9123). This will not persist state. diff --git a/crates/sui/Cargo.toml b/crates/sui/Cargo.toml index b2f8de8d04c71..5d19ba411cc3c 100644 --- a/crates/sui/Cargo.toml +++ b/crates/sui/Cargo.toml @@ -56,8 +56,8 @@ sui-cluster-test.workspace = true sui-execution = { path = "../../sui-execution" } sui-faucet.workspace = true sui-swarm-config.workspace = true -sui-graphql-rpc = {workspace = true, optional = true} -sui-indexer = { workspace = true, optional = true } +sui-graphql-rpc = {workspace = true } +sui-indexer = { workspace = true } sui-genesis-builder.workspace = true sui-types.workspace = true sui-json.workspace = true @@ -133,4 +133,3 @@ gas-profiler = [ "sui-types/gas-profiler", "sui-execution/gas-profiler", ] -indexer = ["dep:sui-indexer", "dep:sui-graphql-rpc"] diff --git a/crates/sui/src/sui_commands.rs b/crates/sui/src/sui_commands.rs index 569007d9c2d5e..0cee79354e625 100644 --- a/crates/sui/src/sui_commands.rs +++ b/crates/sui/src/sui_commands.rs @@ -32,13 +32,13 @@ use sui_config::{ SUI_BENCHMARK_GENESIS_GAS_KEYSTORE_FILENAME, SUI_GENESIS_FILENAME, SUI_KEYSTORE_FILENAME, }; use sui_faucet::{create_wallet_context, start_faucet, AppState, FaucetConfig, SimpleFaucet}; -#[cfg(feature = "indexer")] +use sui_indexer::test_utils::{start_test_indexer, ReaderWriterConfig}; + use sui_graphql_rpc::{ config::{ConnectionConfig, ServiceConfig}, test_infra::cluster::start_graphql_server_with_fn_rpc, }; -#[cfg(feature = "indexer")] -use sui_indexer::test_utils::{start_test_indexer, ReaderWriterConfig}; + use sui_keys::keypair_file::read_key; use sui_keys::keystore::{AccountKeystore, FileBasedKeystore, Keystore}; use sui_move::{self, execute_move_command}; @@ -61,21 +61,19 @@ const DEFAULT_EPOCH_DURATION_MS: u64 = 60_000; const DEFAULT_FAUCET_NUM_COINS: usize = 5; // 5 coins per request was the default in sui-test-validator const DEFAULT_FAUCET_MIST_AMOUNT: u64 = 200_000_000_000; // 200 SUI const DEFAULT_FAUCET_PORT: u16 = 9123; -#[cfg(feature = "indexer")] + const DEFAULT_GRAPHQL_PORT: u16 = 9125; -#[cfg(feature = "indexer")] + const DEFAULT_INDEXER_PORT: u16 = 9124; -#[cfg(feature = "indexer")] #[derive(Args)] -pub struct IndexerFeatureArgs { +pub struct IndexerArgs { /// Start an indexer with default host and port: 0.0.0.0:9124. This flag accepts also a port, /// a host, or both (e.g., 0.0.0.0:9124). /// When providing a specific value, please use the = sign between the flag and value: /// `--with-indexer=6124` or `--with-indexer=0.0.0.0`, or `--with-indexer=0.0.0.0:9124` /// The indexer will be started in writer mode and reader mode. #[clap(long, - default_value = "0.0.0.0:9124", default_missing_value = "0.0.0.0:9124", num_args = 0..=1, require_equals = true, @@ -119,8 +117,7 @@ pub struct IndexerFeatureArgs { pg_password: String, } -#[cfg(feature = "indexer")] -impl IndexerFeatureArgs { +impl IndexerArgs { pub fn for_testing() -> Self { Self { with_indexer: None, @@ -178,9 +175,8 @@ pub enum SuiCommand { )] with_faucet: Option, - #[cfg(feature = "indexer")] #[clap(flatten)] - indexer_feature_args: IndexerFeatureArgs, + indexer_feature_args: IndexerArgs, /// Port to start the Fullnode RPC server on. Default port is 9000. #[clap(long, default_value = "9000")] @@ -354,7 +350,7 @@ impl SuiCommand { config_dir, force_regenesis, with_faucet, - #[cfg(feature = "indexer")] + indexer_feature_args, fullnode_rpc_port, no_full_node, @@ -363,7 +359,6 @@ impl SuiCommand { start( config_dir.clone(), with_faucet, - #[cfg(feature = "indexer")] indexer_feature_args, force_regenesis, epoch_duration_ms, @@ -567,7 +562,7 @@ impl SuiCommand { async fn start( config: Option, with_faucet: Option, - #[cfg(feature = "indexer")] indexer_feature_args: IndexerFeatureArgs, + indexer_feature_args: IndexerArgs, force_regenesis: bool, epoch_duration_ms: Option, fullnode_rpc_port: u16, @@ -580,8 +575,7 @@ async fn start( ); } - #[cfg(feature = "indexer")] - let IndexerFeatureArgs { + let IndexerArgs { mut with_indexer, with_graphql, pg_port, @@ -591,12 +585,12 @@ async fn start( pg_password, } = indexer_feature_args; - #[cfg(feature = "indexer")] + let pg_address = format!("postgres://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_db_name}"); + if with_graphql.is_some() { with_indexer = Some(with_indexer.unwrap_or_default()); } - #[cfg(feature = "indexer")] if with_indexer.is_some() { ensure!( !no_full_node, @@ -661,13 +655,12 @@ async fn start( .with_network_config(network_config); } - #[cfg(feature = "indexer")] let data_ingestion_path = tempdir()?.into_path(); // the indexer requires to set the fullnode's data ingestion directory // note that this overrides the default configuration that is set when running the genesis // command, which sets data_ingestion_dir to None. - #[cfg(feature = "indexer")] + if with_indexer.is_some() { swarm_builder = swarm_builder.with_data_ingestion_dir(data_ingestion_path.clone()); } @@ -692,36 +685,31 @@ async fn start( // the indexer requires a fullnode url with protocol specified let fullnode_url = format!("http://{}", fullnode_url); info!("Fullnode URL: {}", fullnode_url); - #[cfg(feature = "indexer")] - let pg_address = format!("postgres://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_db_name}"); - #[cfg(feature = "indexer")] if let Some(input) = with_indexer { let indexer_address = parse_host_port(input, DEFAULT_INDEXER_PORT) .map_err(|_| anyhow!("Invalid indexer host and port"))?; - tracing::info!("Starting the indexer service at {indexer_address}"); - // Start in writer mode + info!("Starting the indexer service at {indexer_address}"); + // Start in reader mode start_test_indexer( pg_address.clone(), fullnode_url.clone(), - ReaderWriterConfig::writer_mode(None, None), + ReaderWriterConfig::reader_mode(indexer_address.to_string()), data_ingestion_path.clone(), ) .await; - info!("Indexer in writer mode started"); - - // Start in reader mode + info!("Indexer started in reader mode"); + // Start in writer mode start_test_indexer( pg_address.clone(), fullnode_url.clone(), - ReaderWriterConfig::reader_mode(indexer_address.to_string()), - data_ingestion_path, + ReaderWriterConfig::writer_mode(None, None), + data_ingestion_path.clone(), ) .await; - info!("Indexer in reader mode started"); + info!("Indexer started in writer mode"); } - #[cfg(feature = "indexer")] if let Some(input) = with_graphql { let graphql_address = parse_host_port(input, DEFAULT_GRAPHQL_PORT) .map_err(|_| anyhow!("Invalid graphql host and port"))?; diff --git a/crates/sui/tests/cli_tests.rs b/crates/sui/tests/cli_tests.rs index 7d0c374dce9df..c02a35dead329 100644 --- a/crates/sui/tests/cli_tests.rs +++ b/crates/sui/tests/cli_tests.rs @@ -16,8 +16,7 @@ use move_package::{lock_file::schema::ManagedPackage, BuildConfig as MoveBuildCo use serde_json::json; use sui::client_ptb::ptb::PTB; use sui::key_identity::{get_identity_address, KeyIdentity}; -#[cfg(feature = "indexer")] -use sui::sui_commands::IndexerFeatureArgs; +use sui::sui_commands::IndexerArgs; use sui_sdk::SuiClient; use sui_test_transaction_builder::batch_make_transfer_transactions; use sui_types::object::Owner; @@ -76,8 +75,7 @@ async fn test_genesis() -> Result<(), anyhow::Error> { fullnode_rpc_port: 9000, epoch_duration_ms: None, no_full_node: false, - #[cfg(feature = "indexer")] - indexer_feature_args: IndexerFeatureArgs::for_testing(), + indexer_feature_args: IndexerArgs::for_testing(), } .execute() .await; diff --git a/docs/content/guides/developer/getting-started/local-network.mdx b/docs/content/guides/developer/getting-started/local-network.mdx index 322755954cc70..afbf5f6cacaf5 100644 --- a/docs/content/guides/developer/getting-started/local-network.mdx +++ b/docs/content/guides/developer/getting-started/local-network.mdx @@ -2,7 +2,7 @@ title: Connect to a Local Network --- -Use a Sui local network to test your dApps against the latest changes to Sui, and to prepare for the next Sui release to the Devnet or Testnet network. Sui CLI provides the `sui start` command to start a local network. There are several services that can be started when using `sui start`, such as an indexer, a faucet, or a local instance of the GraphQL service (including the online GraphiQL IDE). You can use the included faucet to get test SUI to use on the local network. +Use a Sui local network to test your dApps against the latest changes to Sui, and to prepare for the next Sui release to the Devnet or Testnet network. Sui CLI provides the `sui start` command to start a local network. There are several services that can be started when using `sui start`, such as an indexer, a faucet, or a local instance of the GraphQL service (including the web-based GraphiQL IDE). You can use the included faucet to get test SUI to use on the local network. If you haven't already, you need to [install Sui CLI](./sui-install.mdx) on your system. @@ -27,7 +27,7 @@ Each time you start the network by passing `--force-regenesis`, the local networ To customize your local Sui network, such as starting other services or changing default ports and hosts, include additional flags or options in the `sui start` command. :::info -Options and flags like `with-indexer`, `with-graphql`, and related require you to use the `sui-pg` binary from the [GitHub release archive](https://github.com/MystenLabs/sui/releases), or to build the `sui` binary with the indexer feature enabled: `cargo build --bin sui --features indexer`. +Options and flags like `with-indexer`, `with-graphql`, and related require you to have Postgresq/libpq installed. Check out the list of possible options below to find which is the default expected DB, or how to pass a different DB. ::: The following is a list of possible options and flags to pass to `sui start`: diff --git a/sdk/graphql-transport/package.json b/sdk/graphql-transport/package.json index 53b138b4a679b..e16717d36bf36 100644 --- a/sdk/graphql-transport/package.json +++ b/sdk/graphql-transport/package.json @@ -31,7 +31,7 @@ "prettier:check": "prettier -c --ignore-unknown .", "prettier:fix": "prettier -w --ignore-unknown .", "test:e2e:nowait": "vitest run e2e", - "test:e2e:prepare": "docker-compose down && docker-compose up -d && cargo build --bin sui --features indexer --profile dev && cross-env RUST_LOG=info,sui=error,anemo_tower=warn,consensus=off cargo run --features indexer --bin sui -- start --with-faucet --force-regenesis --with-indexer --pg-port 5435 --pg-db-name sui_indexer_v2 --with-graphql", + "test:e2e:prepare": "docker-compose down && docker-compose up -d && cargo build --bin sui --profile dev && cross-env RUST_LOG=info,sui=error,anemo_tower=warn,consensus=off cargo run --bin sui -- start --with-faucet --force-regenesis --with-indexer --pg-port 5435 --pg-db-name sui_indexer_v2 --with-graphql", "test:e2e": "wait-on http://127.0.0.1:9123 -l --timeout 180000 && vitest" }, "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index d135c3d72a43d..4c8cc66d6e0ee 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -102,7 +102,7 @@ "test:unit": "vitest run unit __tests__", "test:e2e": "wait-on http://127.0.0.1:9123 -l --timeout 180000 && vitest run e2e", "test:e2e:nowait": "vitest run e2e", - "prepare:e2e": "docker-compose down && docker-compose up -d && cargo build --bin sui --features indexer --profile dev && cross-env RUST_LOG=warn,sui=error,anemo_tower=warn,consensus=off ../../target/debug/sui start --with-faucet --force-regenesis --with-indexer --pg-port 5435 --pg-db-name sui_indexer_v2 --with-graphql", + "prepare:e2e": "docker-compose down && docker-compose up -d && cargo build --bin sui --profile dev && cross-env RUST_LOG=warn,sui=error,anemo_tower=warn,consensus=off ../../target/debug/sui start --with-faucet --force-regenesis --with-indexer --pg-port 5435 --pg-db-name sui_indexer_v2 --with-graphql", "prepublishOnly": "pnpm build", "size": "size-limit", "analyze": "size-limit --why",