indexer-alt: Alternative indexer architecture POC

.github/workflows/rust.yml

@@ -189,6 +189,9 @@ jobs:
       - name: Indexer schema
         run: |
           ./scripts/generate_indexer_schema.sh
+      - name: Indexer Alt schema
+        run: |
+          ./crates/sui-indexer-alt/generate_schema.sh
       # Ensure there are no uncommitted changes in the repo after running tests
       - run: scripts/changed-files.sh
         shell: bash

Cargo.toml

@@ -114,6 +114,7 @@ members = [
     "crates/sui-graphql-rpc-client",
     "crates/sui-graphql-rpc-headers",
     "crates/sui-indexer",
+    "crates/sui-indexer-alt",
     "crates/sui-indexer-builder",
     "crates/sui-json",
     "crates/sui-json-rpc",
@@ -540,6 +541,7 @@ webpki = { version = "0.102", package = "rustls-webpki", features = [
     "alloc",
     "std",
 ] }
+wiremock = "0.5"
 x509-parser = "0.14.0"
 zstd = "0.12.3"
 zeroize = "1.6.0"

crates/sui-indexer-alt/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "sui-indexer-alt"
+version.workspace = true
+authors = ["Mysten Labs <[email protected]>"]
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[[bin]]
+name = "sui-indexer-alt"
+path = "src/main.rs"
+
+[dependencies]
+anyhow.workspace = true
+async-trait.workspace = true
+axum.workspace = true
+backoff.workspace = true
+bb8 = "0.8.5"
+bcs.workspace = true
+clap.workspace = true
+diesel = { workspace = true, features = ["chrono"] }
+diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] }
+futures.workspace = true
+prometheus.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+telemetry-subscribers.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tokio-stream.workspace = true
+tokio-util.workspace = true
+tracing.workspace = true
+url.workspace = true
+
+mysten-metrics.workspace = true
+sui-storage.workspace = true
+sui-types.workspace = true
+
+[dev-dependencies]
+rand.workspace = true
+wiremock.workspace = true
+
+sui-types = { workspace = true, features = ["test-utils"] }

crates/sui-indexer-alt/diesel.toml

@@ -0,0 +1,6 @@
+[print_schema]
+file = "src/schema.rs"
+patch_file = "schema.patch"
+
+[migrations_directory]
+dir = "migrations"

crates/sui-indexer-alt/generate_schema.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# Copyright (c) Mysten Labs, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Update sui-indexer's generated src/schema.rs based on the schema after
+# running all its migrations on a clean database. Expects the first argument to
+# be a port to run the temporary database on (defaults to 5433).
+
+set -x
+set -e
+
+if ! command -v git &> /dev/null; then
+    echo "Please install git: e.g. brew install git" >&2
+    exit 1
+fi
+
+for PG in psql initdb postgres pg_isready pg_ctl; do
+    if ! command -v $PG &> /dev/null; then
+        echo "Could not find $PG. Please install postgres: e.g. brew install postgresql@15" >&2
+        exit 1
+    fi
+done
+
+if ! command -v diesel &> /dev/null; then
+    echo "Please install diesel: e.g. cargo install diesel_cli --features postgres" >&2
+    exit 1
+fi
+
+REPO=$(git rev-parse --show-toplevel)
+
+# Create a temporary directory to store the ephemeral DB.
+TMP=$(mktemp -d)
+
+# Set-up a trap to clean everything up on EXIT (stop DB, delete temp directory)
+function cleanup {
+  pg_ctl stop -D "$TMP" -mfast
+  set +x
+  echo "Postgres STDOUT:"
+  cat "$TMP/db.stdout"
+  echo "Postgres STDERR:"
+  cat "$TMP/db.stderr"
+  set -x
+  rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+# Create a new database in the temporary directory
+initdb -D "$TMP" --user postgres
+
+# Run the DB in the background, on the port provided and capture its output
+PORT=${1:-5433}
+postgres -D "$TMP" -p "$PORT" -c unix_socket_directories=                      \
+   > "$TMP/db.stdout"                                                          \
+  2> "$TMP/db.stderr"                                                          &
+
+# Wait for postgres to report as ready
+RETRIES=0
+while ! pg_isready -p "$PORT" --host "localhost" --username "postgres"; do
+  if [ $RETRIES -gt 5 ]; then
+    echo "Postgres failed to start" >&2
+    exit 1
+  fi
+  sleep 1
+  RETRIES=$((RETRIES + 1))
+done
+
+# Run all migrations on the new database
+diesel migration run                                                          \
+  --database-url "postgres://postgres:postgrespw@localhost:$PORT"             \
+  --migration-dir "$REPO/crates/sui-indexer-alt/migrations"
+
+# Generate the schema.rs file, excluding partition tables and including the
+# copyright notice.
+diesel print-schema                                                           \
+  --database-url "postgres://postgres:postgrespw@localhost:$PORT"             \
+  --patch-file "$REPO/crates/sui-indexer-alt/schema.patch"                    \
+  > "$REPO/crates/sui-indexer-alt/src/schema.rs"

crates/sui-indexer-alt/migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();