diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1dda2027b8ebb..0f9a0679bb68d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -156,6 +156,8 @@ jobs: with: ref: ${{ github.event.inputs.sui_repo_ref || github.ref }} - uses: taiki-e/install-action@nextest + - name: Add postgres to PATH + run: echo "/usr/lib/postgresql/14/bin" >> $GITHUB_PATH - name: Set Swap Space uses: pierotofy/set-swap-space@master with: @@ -179,6 +181,14 @@ jobs: - name: sui-execution run: | ./scripts/execution_layer.py generate-lib + - name: Install diesel CLI, and cache the binary + uses: baptiste0928/cargo-install@1cd874a5478fdca35d868ccc74640c5aabbb8f1b # pin@v3.0.0 + with: + crate: diesel_cli + locked: true + - name: Indexer schema + run: | + ./scripts/generate_indexer_schema.sh # Ensure there are no uncommitted changes in the repo after running tests - run: scripts/changed-files.sh shell: bash diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema.rs similarity index 84% rename from crates/sui-indexer/src/schema/pg.rs rename to crates/sui-indexer/src/schema.rs index 540fb9c14f876..21c809dcdd7d2 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema.rs @@ -27,7 +27,7 @@ diesel::table! { validator_signature -> Bytea, end_of_epoch_data -> Nullable, min_tx_sequence_number -> Nullable, - max_tx_sequence_number -> Nullable + max_tx_sequence_number -> Nullable, } } @@ -146,20 +146,6 @@ diesel::table! { } } -diesel::table! { - events_partition_0 (tx_sequence_number, event_sequence_number) { - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - transaction_digest -> Bytea, - senders -> Array>, - package -> Bytea, - module -> Text, - event_type -> Text, - timestamp_ms -> Int8, - bcs -> Bytea, - } -} - diesel::table! { feature_flags (protocol_version, flag_name) { protocol_version -> Int8, @@ -221,29 +207,6 @@ diesel::table! { } } -diesel::table! { - objects_history_partition_0 (checkpoint_sequence_number, object_id, object_version) { - object_id -> Bytea, - object_version -> Int8, - object_status -> Int2, - object_digest -> Nullable, - checkpoint_sequence_number -> Int8, - owner_type -> Nullable, - owner_id -> Nullable, - object_type -> Nullable, - object_type_package -> Nullable, - object_type_module -> Nullable, - object_type_name -> Nullable, - serialized_object -> Nullable, - coin_type -> Nullable, - coin_balance -> Nullable, - df_kind -> Nullable, - df_name -> Nullable, - df_object_type -> Nullable, - df_object_id -> Nullable, - } -} - diesel::table! { objects_snapshot (object_id) { object_id -> Bytea, @@ -275,14 +238,6 @@ diesel::table! { } } -diesel::table! { - protocol_configs (protocol_version, config_name) { - protocol_version -> Int8, - config_name -> Text, - config_value -> Nullable, - } -} - diesel::table! { packages (package_id, original_id, package_version) { package_id -> Bytea, @@ -294,31 +249,23 @@ diesel::table! { } diesel::table! { - pruner_cp_watermark (checkpoint_sequence_number) { - checkpoint_sequence_number -> Int8, - min_tx_sequence_number -> Int8, - max_tx_sequence_number -> Int8, + protocol_configs (protocol_version, config_name) { + protocol_version -> Int8, + config_name -> Text, + config_value -> Nullable, } } diesel::table! { - transactions (tx_sequence_number) { - tx_sequence_number -> Int8, - transaction_digest -> Bytea, - raw_transaction -> Bytea, - raw_effects -> Bytea, + pruner_cp_watermark (checkpoint_sequence_number) { checkpoint_sequence_number -> Int8, - timestamp_ms -> Int8, - object_changes -> Array>, - balance_changes -> Array>, - events -> Array>, - transaction_kind -> Int2, - success_command_count -> Int2, + min_tx_sequence_number -> Int8, + max_tx_sequence_number -> Int8, } } diesel::table! { - transactions_partition_0 (tx_sequence_number) { + transactions (tx_sequence_number) { tx_sequence_number -> Int8, transaction_digest -> Bytea, raw_transaction -> Bytea, @@ -418,18 +365,16 @@ diesel::allow_tables_to_appear_in_same_query!( event_struct_name, event_struct_package, events, - events_partition_0, feature_flags, + full_objects_history, objects, objects_history, - objects_history_partition_0, objects_snapshot, objects_version, packages, protocol_configs, pruner_cp_watermark, transactions, - transactions_partition_0, tx_calls_fun, tx_calls_mod, tx_calls_pkg, diff --git a/crates/sui-indexer/src/schema/mod.rs b/crates/sui-indexer/src/schema/mod.rs deleted file mode 100644 index b65995cbcc233..0000000000000 --- a/crates/sui-indexer/src/schema/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -#![allow(clippy::all)] - -mod pg; - -pub use pg::chain_identifier; -pub use pg::checkpoints; -pub use pg::display; -pub use pg::epochs; -pub use pg::event_emit_module; -pub use pg::event_emit_package; -pub use pg::event_senders; -pub use pg::event_struct_instantiation; -pub use pg::event_struct_module; -pub use pg::event_struct_name; -pub use pg::event_struct_package; -pub use pg::events; -pub use pg::feature_flags; -pub use pg::full_objects_history; -pub use pg::objects; -pub use pg::objects_history; -pub use pg::objects_snapshot; -pub use pg::objects_version; -pub use pg::packages; -pub use pg::protocol_configs; -pub use pg::pruner_cp_watermark; -pub use pg::transactions; -pub use pg::tx_calls_fun; -pub use pg::tx_calls_mod; -pub use pg::tx_calls_pkg; -pub use pg::tx_changed_objects; -pub use pg::tx_digests; -pub use pg::tx_input_objects; -pub use pg::tx_kinds; -pub use pg::tx_recipients; -pub use pg::tx_senders; - -pub use pg::events_partition_0; -pub use pg::objects_history_partition_0; -pub use pg::transactions_partition_0; diff --git a/scripts/generate_indexer_schema.sh b/scripts/generate_indexer_schema.sh new file mode 100755 index 0000000000000..f505cd65814e5 --- /dev/null +++ b/scripts/generate_indexer_schema.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Update sui-indexer's generated src/schema.rs based on the schema after +# running all its migrations on a clean database. Expects the first argument to +# be a port to run the temporary database on (defaults to 5433). + +set -x +set -e + +if ! command -v git &> /dev/null; then + echo "Please install git: e.g. brew install git" >&2 + exit 1 +fi + +for PG in psql initdb postgres pg_isready pg_ctl; do + if ! command -v $PG &> /dev/null; then + echo "Could not find $PG. Please install postgres: e.g. brew install postgresql@15" >&2 + exit 1 + fi +done + +if ! command -v diesel &> /dev/null; then + echo "Please install diesel: e.g. cargo install diesel_cli --features postgres" >&2 + exit 1 +fi + +REPO=$(git rev-parse --show-toplevel) + +# Create a temporary directory to store the ephemeral DB. +TMP=$(mktemp -d) + +# Set-up a trap to clean everything up on EXIT (stop DB, delete temp directory) +function cleanup { + pg_ctl stop -D "$TMP" -mfast + set +x + echo "Postgres STDOUT:" + cat "$TMP/db.stdout" + echo "Postgres STDERR:" + cat "$TMP/db.stderr" + set -x + rm -rf "$TMP" +} +trap cleanup EXIT + +# Create a new database in the temporary directory +initdb -D "$TMP" --user postgres + +# Run the DB in the background, on the port provided and capture its output +PORT=${1:-5433} +postgres -D "$TMP" -p "$PORT" -c unix_socket_directories= \ + > "$TMP/db.stdout" \ + 2> "$TMP/db.stderr" & + +# Wait for postgres to report as ready +RETRIES=0 +while ! pg_isready -p "$PORT" --host "localhost" --username "postgres"; do + if [ $RETRIES -gt 5 ]; then + echo "Postgres failed to start" >&2 + exit 1 + fi + sleep 1 + RETRIES=$((RETRIES + 1)) +done + +# Run all migrations on the new database +diesel migration run \ + --database-url "postgres://postgres:postgrespw@localhost:$PORT" \ + --migration-dir "$REPO/crates/sui-indexer/migrations/pg" + +# Generate the schema.rs file, excluding partition tables and including the +# copyright notice. +diesel print-schema \ + --database-url "postgres://postgres:postgrespw@localhost:$PORT" \ + --patch-file "$REPO/crates/sui-indexer/src/schema.patch" \ + --except-tables "^objects_version_|_partition_" \ + > "$REPO/crates/sui-indexer/src/schema.rs"