diff --git a/build/os_latest.json.gz b/build/os_latest.json.gz new file mode 100644 index 00000000..b69594c0 Binary files /dev/null and b/build/os_latest.json.gz differ diff --git a/crates/bin/hint_tool/Cargo.toml b/crates/bin/hint_tool/Cargo.toml index 282ba3f3..51deebed 100644 --- a/crates/bin/hint_tool/Cargo.toml +++ b/crates/bin/hint_tool/Cargo.toml @@ -9,6 +9,7 @@ license-file.workspace = true blockifier = { workspace = true } cairo-vm = { workspace = true } clap = { workspace = true } +flate2 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } starknet-os = { path = "../../starknet-os" } diff --git a/crates/bin/hint_tool/src/main.rs b/crates/bin/hint_tool/src/main.rs index a47ef427..c679da80 100644 --- a/crates/bin/hint_tool/src/main.rs +++ b/crates/bin/hint_tool/src/main.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; +use std::ffi::OsStr; use std::fs::File; -use std::io::{BufReader, Write}; +use std::io::{BufReader, Read as _, Write}; use std::path::PathBuf; use blockifier::execution::hint_code; @@ -12,6 +13,7 @@ use cairo_vm::vm::errors::hint_errors::HintError; use cairo_vm::vm::vm_core::VirtualMachineBuilder; use cairo_vm::vm::vm_memory::memory_segments::MemorySegmentManager; use clap::Parser; +use flate2::read::GzDecoder; use serde::Deserialize; use starknet_os::crypto::pedersen::PedersenHash; use starknet_os::hints::SnosHintProcessor; @@ -54,9 +56,10 @@ struct Args { #[arg(long)] out_file: Option, - /// Input JSON file (e.g. "os_latest.json") + /// Input JSON file, optionally gzipped (e.g. "os_latest.json.gz"). Will be transparently + /// decormpressed if needed. #[arg(long)] - #[clap(default_value = Some("../../build/os_latest.json"))] + #[clap(default_value = Some("../../../build/os_latest.json.gz"))] in_file: PathBuf, } @@ -79,8 +82,21 @@ fn main() -> std::io::Result<()> { let mut result = Vec::new(); - let os: Os = - serde_json::from_reader(BufReader::new(File::open(args.in_file)?)).expect("Failed to parse os_latest.json"); + // load the os JSON. if it ends with 'gz' we will attempt to gunzip it + let os_buf = if args.in_file.extension() == Some(OsStr::new("gz")) { + let compressed_buf = BufReader::new(File::open(args.in_file)?); + let mut decoder = GzDecoder::new(compressed_buf); + let mut os_gunzipped = Vec::::new(); + let _ = decoder.read_to_end(&mut os_gunzipped)?; + os_gunzipped + } else { + let mut os_raw = Vec::::new(); + let mut reader = BufReader::new(File::open(args.in_file)?); + let _ = reader.read_to_end(&mut os_raw)?; + os_raw + }; + + let os: Os = serde_json::from_slice(os_buf.as_slice()).expect("Failed to parse os_latest.json"); let os_hints = os.hints.into_values().flatten().map(|h| h.code.to_string()).collect::>(); let syscall_hints = hint_code::SYSCALL_HINTS.into_iter().map(|h| h.to_string()).collect::>(); diff --git a/crates/bin/prove_block/src/lib.rs b/crates/bin/prove_block/src/lib.rs index 5b431189..50f83df4 100644 --- a/crates/bin/prove_block/src/lib.rs +++ b/crates/bin/prove_block/src/lib.rs @@ -324,7 +324,8 @@ pub async fn prove_block( (old_block_number, old_block_hash), ); - Ok(run_os(config::DEFAULT_COMPILED_OS, layout, os_input, block_context, execution_helper)?) + let uncompressed_os = config::gunzip_default_os().expect("Could not uncompress default OS"); + Ok(run_os(uncompressed_os.as_slice(), layout, os_input, block_context, execution_helper)?) } pub fn debug_prove_error(err: ProveBlockError) -> ProveBlockError { diff --git a/crates/starknet-os/Cargo.toml b/crates/starknet-os/Cargo.toml index c6fd0497..56bac792 100644 --- a/crates/starknet-os/Cargo.toml +++ b/crates/starknet-os/Cargo.toml @@ -20,6 +20,7 @@ cairo-lang-casm = { workspace = true } cairo-type-derive = { path = "../cairo-type-derive" } cairo-vm = { workspace = true } c-kzg = { workspace = true } +flate2 = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } heck = { workspace = true } diff --git a/crates/starknet-os/src/config.rs b/crates/starknet-os/src/config.rs index 01ed8d75..811d5a40 100644 --- a/crates/starknet-os/src/config.rs +++ b/crates/starknet-os/src/config.rs @@ -1,4 +1,5 @@ use std::fs::File; +use std::io::Read as _; use std::path::PathBuf; use blockifier::blockifier::block::{BlockInfo, GasPrices}; @@ -7,6 +8,7 @@ use blockifier::context::{BlockContext, ChainInfo, FeeTokenAddresses}; use blockifier::transaction::objects::FeeType; use blockifier::versioned_constants::VersionedConstants; use cairo_vm::types::layout_name::LayoutName; +use flate2::read::GzDecoder; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use starknet_api::block::{BlockNumber, BlockTimestamp}; @@ -23,7 +25,7 @@ pub const fn default_layout() -> LayoutName { // https://github.com/starkware-libs/blockifier/blob/8da582b285bfbc7d4c21178609bbd43f80a69240/crates/native_blockifier/src/py_block_executor.rs#L44 const MAX_STEPS_PER_TX: u32 = 4_000_000; -pub const DEFAULT_COMPILED_OS: &[u8] = include_bytes!("../../../build/os_latest.json"); +pub const DEFAULT_COMPILED_OS_GZIPPED: &[u8] = include_bytes!("../../../build/os_latest.json.gz"); const DEFAULT_CONFIG_PATH: &str = "../../cairo-lang/src/starkware/starknet/definitions/general_config.yml"; pub const STORED_BLOCK_HASH_BUFFER: u64 = 10; @@ -67,6 +69,24 @@ const fn default_use_kzg_da() -> bool { true } +/// un-gzips the default OS that is baked into the binary. +/// +/// Uncompressed, this file is roughly 40M, and compresses effectively down to less than 1M. +/// So it is included in the binary in gzipped form with this fn provided to reinflate it. +/// +/// NOTE: the results are not cached, so this will always be an expensive call. +pub fn gunzip_default_os() -> std::io::Result> { + let mut decoder = GzDecoder::new(DEFAULT_COMPILED_OS_GZIPPED); + let mut os_gunzipped = Vec::::new(); + let uncompressed_size = decoder.read_to_end(&mut os_gunzipped)?; + log::debug!( + "Os inflated from {}b (gzipped) to {}b (raw json)", + DEFAULT_COMPILED_OS_GZIPPED.len(), + uncompressed_size + ); + Ok(os_gunzipped) +} + #[derive(Debug, Serialize, Clone, Deserialize, PartialEq)] pub struct StarknetGeneralConfig { pub starknet_os_config: StarknetOsConfig, diff --git a/scripts/setup-tests.sh b/scripts/setup-tests.sh index 86a72814..f2aaf826 100755 --- a/scripts/setup-tests.sh +++ b/scripts/setup-tests.sh @@ -44,6 +44,7 @@ cairo-compile tests/integration/programs/fact.cairo --output build/programs/fact # compile os with debug info cairo-compile --debug_info_with_source cairo-lang/src/starkware/starknet/core/os/os.cairo --output build/os_debug.json --cairo_path cairo-lang/src cairo-compile cairo-lang/src/starkware/starknet/core/os/os.cairo --output build/os_latest.json --cairo_path cairo-lang/src +gzip -f build/os_latest.json # compile starknet contract echo -e "compiling starknet contracts...\n" diff --git a/scripts/teardown-tests.sh b/scripts/teardown-tests.sh index 61d20abf..bc4573ee 100755 --- a/scripts/teardown-tests.sh +++ b/scripts/teardown-tests.sh @@ -9,4 +9,4 @@ git submodule update --init # remove compiled contracts echo -e "\nremoving compiled contracts/programs...\n" rm starkware -rm -rf build/!(os_latest.json) +rm -rf build/!(os_latest.json.gz) diff --git a/tests/integration/common/transaction_utils.rs b/tests/integration/common/transaction_utils.rs index c3bf1766..6f2cb45b 100644 --- a/tests/integration/common/transaction_utils.rs +++ b/tests/integration/common/transaction_utils.rs @@ -905,7 +905,8 @@ where .await; let layout = config::default_layout(); - let result = run_os(config::DEFAULT_COMPILED_OS, layout, os_input, block_context, execution_helper); + let uncompressed_os = config::gunzip_default_os().expect("Could not uncompress default OS"); + let result = run_os(uncompressed_os.as_slice(), layout, os_input, block_context, execution_helper); match &result { Err(Runner(VmException(vme))) => {