[pick][GraphQL] ConsensusCommitPrologueV3 (#18844)

## Description

Pick PRs to introduce the `ConsensusCommitPrologueV3` types into the
(soon-to-be legacy) GraphQL release branch, so that it can continue
deserializing transaction blocks and effects.

Cherry picks include: #17678, #18816, #18789 (+ a pick to `fastcrypto`
to use `reqwest 0.12`).

## Test plan

```
sui$ cargo simtest
sui-graphql-rpc$ cargo nextest run
sui-graphql-e2e-tests$ cargo nextest run --features pg_integration
```

and CI

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:

---------

Co-authored-by: Brandon Williams <[email protected]>

Cargo.toml

@@ -237,7 +237,8 @@ opt-level = 1
 # Dependencies that should be kept in sync through the whole workspace
 [workspace.dependencies]
 anyhow = "1.0.71"
-arrow-array = "50.0.0"
+arrow = "52"
+arrow-array = "52"
 arc-swap = { version = "1.5.1", features = ["serde"] }
 assert_cmd = "2.0.6"
 async-graphql = "6.0.7"
@@ -376,8 +377,8 @@ moka = { version = "0.12", default-features = false, features = [
   "atomic64",
 ] }
 more-asserts = "0.3.1"
-msim = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "6f88ec84644cb1a6809c010f1f534d0d09e0cd89", package = "msim" }
-msim-macros = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "6f88ec84644cb1a6809c010f1f534d0d09e0cd89", package = "msim-macros" }
+msim = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "291cebe772727338f474a020e094b4ecb7ba1fe9", package = "msim" }
+msim-macros = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "291cebe772727338f474a020e094b4ecb7ba1fe9", package = "msim-macros" }
 multiaddr = "0.17.0"
 nexlint = { git = "https://github.com/nextest-rs/nexlint.git", rev = "94da5c787636dad779c340affa65219134d127f5" }
 nexlint-lints = { git = "https://github.com/nextest-rs/nexlint.git", rev = "94da5c787636dad779c340affa65219134d127f5" }
@@ -387,18 +388,18 @@ ntest = "0.9.0"
 num-bigint = "0.4.4"
 num_cpus = "1.15.0"
 num_enum = "0.6.1"
-object_store = { version = "0.7", features = ["aws", "gcp", "azure", "http"] }
+object_store = { version = "0.10", features = ["aws", "gcp", "azure", "http"] }
 once_cell = "1.18.0"
 ouroboros = "0.17"
 parking_lot = "0.12.1"
-parquet = "50.0.0"
+parquet = "52"
 pkcs8 = { version = "0.9.0", features = ["std"] }
 pprof = { version = "0.11.0", features = ["cpp", "frame-pointer"] }
 pretty_assertions = "1.3.0"
 prettytable-rs = "0.10.0"
 proc-macro2 = "1.0.47"
 prometheus = "0.13.3"
-prometheus-http-query = { version = "0.6.6", default_features = false, features = [
+prometheus-http-query = { version = "0.8", default_features = false, features = [
   "rustls-tls",
 ] }
 prometheus-parse = { git = "https://github.com/asonnino/prometheus-parser.git", rev = "75334db" }
@@ -414,7 +415,7 @@ rand = "0.8.5"
 rayon = "1.5.3"
 rcgen = "0.9.2"
 regex = "1.7.1"
-reqwest = { version = "0.11.20", default_features = false, features = [
+reqwest = { version = "0.12", default_features = false, features = [
   "blocking",
   "json",
   "rustls-tls",
@@ -465,6 +466,7 @@ similar = "2.4.0"
 slip10_ed25519 = "0.1.3"
 smallvec = "1.10.0"
 snap = "1.1.0"
+snowflake-api = "0.9.0"
 static_assertions = "1.1.0"
 strum = { version = "0.24", features = ["derive"] }
 strum_macros = "0.24.3"
@@ -503,7 +505,6 @@ tower-http = { version = "0.3.4", features = [
   "set-header",
   "propagate-header",
 ] }
-# tower-http = { version="0.4", features = ["trace"] }
 tower-layer = "0.3.2"
 twox-hash = "1.6.3"
 tracing = "0.1.37"
@@ -522,7 +523,7 @@ ttl_cache = "0.5.1"
 uint = "0.9.4"
 unescape = "0.1.0"
 ureq = "2.9.1"
-url = "2.3.1"
+url = "2.5"
 uuid = { version = "1.1.2", features = ["v4", "fast-rng"] }
 webpki = { version = "0.101.0", package = "rustls-webpki", features = [
   "alloc",
@@ -560,9 +561,9 @@ move-stackless-bytecode = { path = "external-crates/move/crates/move-stackless-b
 move-symbol-pool = { path = "external-crates/move/crates/move-symbol-pool" }
 move-abstract-stack = { path = "external-crates/move/crates/move-abstract-stack" }
 
-fastcrypto = { git = "https://github.com/MystenLabs/fastcrypto", rev = "878492bd2541dce1491791bcadf3cc855ddcdc05" }
-fastcrypto-tbls = { git = "https://github.com/MystenLabs/fastcrypto", rev = "878492bd2541dce1491791bcadf3cc855ddcdc05" }
-fastcrypto-zkp = { git = "https://github.com/MystenLabs/fastcrypto", rev = "878492bd2541dce1491791bcadf3cc855ddcdc05", package = "fastcrypto-zkp" }
+fastcrypto = { git = "https://github.com/MystenLabs/fastcrypto", rev = "d72e9a37037a3274ecc76b262f4fe66915eca35b" }
+fastcrypto-tbls = { git = "https://github.com/MystenLabs/fastcrypto", rev = "d72e9a37037a3274ecc76b262f4fe66915eca35b" }
+fastcrypto-zkp = { git = "https://github.com/MystenLabs/fastcrypto", rev = "d72e9a37037a3274ecc76b262f4fe66915eca35b" }
 
 # anemo dependencies
 anemo = { git = "https://github.com/mystenlabs/anemo.git", rev = "26d415eb9aa6a2417be3c03c57d6e93c30bd1ad7" }

crates/mysten-service-boilerplate/Cargo-ext.toml

@@ -8,7 +8,7 @@ publish = false
 
 [dependencies]
 anyhow = "1.0.79"
-axum = { version = "0.6.6", features = ["macros"] }
+axum = { version = "0.7", features = ["macros"] }
 mysten-service = { git = "https://github.com/mystenlabs/sui.git", branch = "main", package = "mysten-service" }
 prometheus = "0.13.3"
 tracing = "0.1.40"

crates/sui-analytics-indexer/Cargo.toml

@@ -51,10 +51,10 @@ move-bytecode-utils.workspace = true
 sui-json-rpc-types.workspace = true
 sui-package-resolver.workspace = true
 simulacrum.workspace = true
-arrow = { version = "50.0.0"}
+arrow.workspace = true
 gcp-bigquery-client = "0.18.0"
-snowflake-api = { version = "0.7.0"  }
-tap = { version = "1.0.1", features = [] }
+snowflake-api.workspace = true
+tap.workspace = true
 
 [dev-dependencies]
 

crates/sui-core/src/checkpoints/mod.rs

@@ -1186,6 +1186,7 @@ impl CheckpointBuilder {
                     transaction.inner().transaction_data().kind(),
                     TransactionKind::ConsensusCommitPrologue(_)
                         | TransactionKind::ConsensusCommitPrologueV2(_)
+                        | TransactionKind::ConsensusCommitPrologueV3(_)
                         | TransactionKind::AuthenticatorStateUpdate(_)
                         | TransactionKind::RandomnessStateUpdate(_)
                 ) {

crates/sui-core/tests/staged/sui.yaml

@@ -237,6 +237,29 @@ ConsensusCommitPrologueV2:
     - commit_timestamp_ms: U64
     - consensus_commit_digest:
         TYPENAME: ConsensusCommitDigest
+ConsensusCommitPrologueV3:
+  STRUCT:
+    - epoch: U64
+    - round: U64
+    - sub_dag_index:
+        OPTION: U64
+    - commit_timestamp_ms: U64
+    - consensus_commit_digest:
+        TYPENAME: ConsensusCommitDigest
+    - consensus_determined_version_assignments:
+        TYPENAME: ConsensusDeterminedVersionAssignments
+ConsensusDeterminedVersionAssignments:
+  ENUM:
+    0:
+      CancelledTransactions:
+        NEWTYPE:
+          SEQ:
+            TUPLE:
+              - TYPENAME: TransactionDigest
+              - SEQ:
+                  TUPLE:
+                    - TYPENAME: ObjectID
+                    - TYPENAME: SequenceNumber
 Data:
   ENUM:
     0:
@@ -973,6 +996,10 @@ TransactionKind:
       ConsensusCommitPrologueV2:
         NEWTYPE:
           TYPENAME: ConsensusCommitPrologueV2
+    8:
+      ConsensusCommitPrologueV3:
+        NEWTYPE:
+          TYPENAME: ConsensusCommitPrologueV3
 TypeArgumentError:
   ENUM:
     0:

crates/sui-data-ingestion/src/workers/archival.rs

@@ -119,7 +119,7 @@ impl ArchivalWorker {
 
         let bytes = finalize_manifest(manifest)?;
         self.remote_store
-            .put(&Path::from("MANIFEST"), bytes)
+            .put(&Path::from("MANIFEST"), bytes.into())
             .await?;
         Ok(())
     }
@@ -134,7 +134,7 @@ impl ArchivalWorker {
         let mut cursor = Cursor::new(buffer);
         compress(&mut cursor, &mut compressed_buffer)?;
         self.remote_store
-            .put(&location, Bytes::from(compressed_buffer.clone()))
+            .put(&location, Bytes::from(compressed_buffer.clone()).into())
             .await?;
         Ok(Bytes::from(compressed_buffer))
     }

crates/sui-data-ingestion/src/workers/blob.rs

@@ -38,7 +38,9 @@ impl Worker for BlobWorker {
             "{}.chk",
             checkpoint.checkpoint_summary.sequence_number
         ));
-        self.remote_store.put(&location, Bytes::from(bytes)).await?;
+        self.remote_store
+            .put(&location, Bytes::from(bytes).into())
+            .await?;
         Ok(())
     }
 }

crates/sui-graphql-rpc-client/src/lib.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use async_graphql::Value;
-use hyper::header::ToStrError;
+use reqwest::header::ToStrError;
 use serde_json::Number;
 
 pub mod response;

crates/sui-graphql-rpc-client/src/response.rs

@@ -3,8 +3,7 @@
 
 use super::ClientError;
 use async_graphql::{Response, ServerError, Value};
-use axum::http::HeaderName;
-use hyper::HeaderMap;
+use reqwest::header::{HeaderMap, HeaderName};
 use reqwest::Response as ReqwestResponse;
 use serde_json::json;
 use std::{collections::BTreeMap, net::SocketAddr};
@@ -14,8 +13,8 @@ use sui_graphql_rpc_headers::VERSION_HEADER;
 pub struct GraphqlResponse {
     headers: HeaderMap,
     remote_address: Option<SocketAddr>,
-    http_version: hyper::Version,
-    status: hyper::StatusCode,
+    http_version: reqwest::Version,
+    status: reqwest::StatusCode,
     full_response: Response,
 }
 
@@ -39,7 +38,7 @@ impl GraphqlResponse {
     pub fn graphql_version(&self) -> Result<String, ClientError> {
         Ok(self
             .headers
-            .get(&VERSION_HEADER)
+            .get(VERSION_HEADER.as_str())
             .ok_or(ClientError::ServiceVersionHeaderNotFound)?
             .to_str()
             .map_err(|e| ClientError::ServiceVersionHeaderValueInvalidString { error: e })?
@@ -58,11 +57,11 @@ impl GraphqlResponse {
         serde_json::to_string_pretty(&self.full_response).unwrap()
     }
 
-    pub fn http_status(&self) -> hyper::StatusCode {
+    pub fn http_status(&self) -> reqwest::StatusCode {
         self.status
     }
 
-    pub fn http_version(&self) -> hyper::Version {
+    pub fn http_version(&self) -> reqwest::Version {
         self.http_version
     }
 

crates/sui-graphql-rpc-client/src/simple_client.rs

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::ClientError;
-use axum::http::HeaderValue;
-use hyper::header;
+use reqwest::header;
+use reqwest::header::HeaderValue;
 use reqwest::Response;
 use serde_json::Value;
 use std::collections::BTreeMap;
@@ -52,7 +52,10 @@ impl SimpleClient {
         mut headers: Vec<(header::HeaderName, header::HeaderValue)>,
     ) -> Result<GraphqlResponse, ClientError> {
         if get_usage {
-            headers.push((LIMITS_HEADER.clone(), HeaderValue::from_static("true")));
+            headers.push((
+                LIMITS_HEADER.clone().as_str().try_into().unwrap(),
+                HeaderValue::from_static("true"),
+            ));
         }
         GraphqlResponse::from_resp(self.execute_impl(query, variables, headers, false).await?).await
     }

crates/sui-graphql-rpc/src/server/builder.rs

@@ -1080,10 +1080,10 @@ pub mod tests {
         server_builder.build_schema();
 
         let resp = reqwest::get(&url).await.unwrap();
-        assert_eq!(resp.status(), StatusCode::OK);
+        assert_eq!(resp.status(), reqwest::StatusCode::OK);
 
         let url_with_param = format!("{}?max_checkpoint_lag_ms=1", url);
         let resp = reqwest::get(&url_with_param).await.unwrap();
-        assert_eq!(resp.status(), StatusCode::GATEWAY_TIMEOUT);
+        assert_eq!(resp.status(), reqwest::StatusCode::GATEWAY_TIMEOUT);
     }
 }

crates/sui-graphql-rpc/src/types/transaction_block_kind/consensus_commit_prologue.rs

@@ -10,6 +10,8 @@ use sui_types::{
     messages_consensus::{
         ConsensusCommitPrologue as NativeConsensusCommitPrologueTransactionV1,
         ConsensusCommitPrologueV2 as NativeConsensusCommitPrologueTransactionV2,
+        ConsensusCommitPrologueV3 as NativeConsensusCommitPrologueTransactionV3,
+        ConsensusDeterminedVersionAssignments,
     },
 };
 
@@ -22,10 +24,17 @@ use sui_types::{
 pub(crate) struct ConsensusCommitPrologueTransaction {
     epoch: u64,
     round: u64,
+    /// The sub DAG index of the consensus commit. This field will be populated if there
+    /// are multiple consensus commits per round.
+    sub_dag_index: Option<u64>,
     commit_timestamp_ms: CheckpointTimestamp,
     consensus_commit_digest: Option<ConsensusCommitDigest>,
     /// The checkpoint sequence number this was viewed at.
     checkpoint_viewed_at: u64,
+    /// Stores consensus handler determined shared object version assignments for transactions
+    /// within the consensus commit.
+    /// Note that currently it only stores cancelled transactions' version assignments.
+    consensus_determined_version_assignments: Option<ConsensusDeterminedVersionAssignments>,
 }
 
 /// System transaction that runs at the beginning of a checkpoint, and is responsible for setting
@@ -65,9 +74,11 @@ impl ConsensusCommitPrologueTransaction {
         Self {
             epoch: ccp.epoch,
             round: ccp.round,
+            sub_dag_index: None,
             commit_timestamp_ms: ccp.commit_timestamp_ms,
             consensus_commit_digest: None,
             checkpoint_viewed_at,
+            consensus_determined_version_assignments: None,
         }
     }
 
@@ -78,9 +89,28 @@ impl ConsensusCommitPrologueTransaction {
         Self {
             epoch: ccp.epoch,
             round: ccp.round,
+            sub_dag_index: None,
             commit_timestamp_ms: ccp.commit_timestamp_ms,
             consensus_commit_digest: Some(ccp.consensus_commit_digest),
             checkpoint_viewed_at,
+            consensus_determined_version_assignments: None,
+        }
+    }
+
+    pub(crate) fn from_v3(
+        ccp: NativeConsensusCommitPrologueTransactionV3,
+        checkpoint_viewed_at: u64,
+    ) -> Self {
+        Self {
+            epoch: ccp.epoch,
+            round: ccp.round,
+            sub_dag_index: ccp.sub_dag_index,
+            commit_timestamp_ms: ccp.commit_timestamp_ms,
+            consensus_commit_digest: Some(ccp.consensus_commit_digest),
+            checkpoint_viewed_at,
+            consensus_determined_version_assignments: Some(
+                ccp.consensus_determined_version_assignments,
+            ),
         }
     }
 }

crates/sui-graphql-rpc/src/types/transaction_block_kind/mod.rs

@@ -56,6 +56,9 @@ impl TransactionBlockKind {
             K::ConsensusCommitPrologueV2(ccp) => T::ConsensusCommitPrologue(
                 ConsensusCommitPrologueTransaction::from_v2(ccp, checkpoint_viewed_at),
             ),
+            K::ConsensusCommitPrologueV3(ccp) => T::ConsensusCommitPrologue(
+                ConsensusCommitPrologueTransaction::from_v3(ccp, checkpoint_viewed_at),
+            ),
             K::AuthenticatorStateUpdate(asu) => {
                 T::AuthenticatorState(AuthenticatorStateUpdateTransaction {
                     native: asu,

crates/sui-graphql-rpc/tests/e2e_tests.rs

@@ -159,7 +159,7 @@ mod tests {
             .unwrap();
 
         assert_eq!(res.http_status().as_u16(), 200);
-        assert_eq!(res.http_version(), hyper::Version::HTTP_11);
+        assert_eq!(res.http_version(), reqwest::Version::HTTP_11);
         assert!(res.graphql_version().unwrap().len() >= 5);
         assert!(res.errors().is_empty());