test(output): compare python output to rust output (#9)

* test(output): compare python output to rust output

* test(hints): add test where output is diff

* izldf,sqmdq

.github/workflows/ci.yml

@@ -28,4 +28,4 @@ jobs:
         run: rustup show
       - uses: Swatinem/rust-cache@v2
       - name: Run tests
-        run: cargo test
+        run: pip install cairo-lang; cargo test

.gitignore

@@ -1 +1,2 @@
 /target
+build/test.zip

Cargo.toml

@@ -18,6 +18,7 @@ hex = "0.4.3"
 starknet = { git = "https://github.com/xJonathanLEI/starknet-rs" }
 uuid = { version = "1.4.0", features = ["v4", "serde"] }
 reqwest = { version = "0.11.18", features = ["blocking", "json"] }
+pretty_assertions = "1.4.0"
 
 [dev-dependencies]
 rstest = "0.18.2"

tests/common.rs

@@ -37,3 +37,28 @@ pub fn setup_pie() -> CairoPie {
 
     runner.get_cairo_pie(&vm).unwrap()
 }
+
+#[allow(unused)]
+pub fn assert_python_and_rust_output_match(program: &str, mut vm: VirtualMachine) {
+    let mut rs_output = String::new();
+    vm.write_output(&mut rs_output).unwrap();
+    let rs_output = rs_output.split('\n').filter(|&x| !x.is_empty());
+    let python_output = std::process::Command::new("cairo-run")
+        .arg("--layout=small")
+        .arg(format!("--program={program:}"))
+        .arg("--print_output")
+        .output()
+        .expect("failed to run python vm");
+    let python_output = unsafe { std::str::from_utf8_unchecked(&python_output.stdout) }.to_string();
+    let python_output = python_output
+        .split('\n')
+        .into_iter()
+        .skip_while(|&x| x != "Program output:")
+        .skip(1)
+        .filter(|&x| !x.trim().is_empty())
+        .into_iter();
+    for (i, (rs, py)) in rs_output.zip(python_output).enumerate() {
+        let py = py.to_string().trim().to_string();
+        pretty_assertions::assert_eq!(*rs, py, "Output #{i:} is different");
+    }
+}

tests/contracts/different_output.cairo

@@ -0,0 +1,12 @@
+%builtins output
+
+func main(output_ptr: felt*) -> (output_ptr: felt*) {
+    tempvar a = 18;
+    a = [output_ptr], ap++;
+
+    // Use custom hint to print the value of a
+    %{ print(ids.a) %}
+    let output_ptr = output_ptr + 1;
+
+    return(output_ptr = output_ptr);
+}

tests/contracts/hint.cairo

@@ -1,6 +1,12 @@
-func main() {
-    let a = 17;
+%builtins output
+
+func main(output_ptr: felt*) -> (output_ptr: felt*) {
+    tempvar a = 17;
+    a = [output_ptr], ap++;
+
     // Use custom hint to print the value of a
     %{ print(ids.a) %}
-    return ();
-}
+    let output_ptr = output_ptr + 1;
+
+    return(output_ptr = output_ptr);
+}

tests/sharp.rs

@@ -14,6 +14,7 @@ const TEST_CAIRO_JOB_ID: &str = "3a24bbca-ad75-49d5-8ced-12796c6c0738";
 // }
 
 #[rstest]
+#[ignore]
 fn sharp_client_status() {
     let sharp_client = SharpClient::default();
     let submit_resp = sharp_client.get_status(TEST_CAIRO_JOB_ID).unwrap();

tests/snos.rs

@@ -1,3 +1,4 @@
+mod common;
 use cairo_vm::cairo_run::{cairo_run, CairoRunConfig};
 use cairo_vm::felt::Felt252;
 use cairo_vm::hint_processor::builtin_hint_processor::builtin_hint_processor_definition::{
@@ -8,6 +9,7 @@ use cairo_vm::hint_processor::hint_processor_definition::HintReference;
 use cairo_vm::serde::deserialize_program::ApTracking;
 use cairo_vm::types::exec_scope::ExecutionScopes;
 use cairo_vm::vm::{errors::hint_errors::HintError, vm_core::VirtualMachine};
+use common::assert_python_and_rust_output_match;
 use snos::SnOsRunner;
 use std::collections::HashMap;
 use std::fs::File;
@@ -30,6 +32,7 @@ fn print_a_hint(
 
 #[test]
 fn custom_hint_ok() {
+    let program_path = "build/hint.json";
     // Wrap the Rust hint implementation in a Box smart pointer inside a HintFunc
     let hint = HintFunc(Box::new(print_a_hint));
 
@@ -39,13 +42,13 @@ fn custom_hint_ok() {
     //Add the custom hint, together with the Python code
     hint_processor.add_hint(String::from("print(ids.a)"), Rc::new(hint));
 
-    let file = File::open(Path::new("build/hint.json")).expect("Couldn't load file");
+    let file = File::open(Path::new(program_path)).expect("Couldn't load file");
     let mut reader = BufReader::new(file);
     let mut buffer = Vec::<u8>::new();
     reader.read_to_end(&mut buffer).expect("Couldn't read file");
 
     //Run the cairo program
-    cairo_run(
+    let (_cairo_runner, virtual_machine) = cairo_run(
         &buffer,
         &CairoRunConfig {
             layout: "all_cairo",
@@ -54,6 +57,7 @@ fn custom_hint_ok() {
         &mut hint_processor,
     )
     .expect("Couldn't run program");
+    assert_python_and_rust_output_match(program_path, virtual_machine);
 }
 
 #[test]
@@ -62,3 +66,34 @@ fn snos_ok() {
     let _runner_res = snos_runner.run();
     assert_eq!(4, 4);
 }
+
+#[test]
+#[should_panic(expected = "Output #0 is different")]
+fn test_different_outputs() {
+    let program_path = "build/hint.json";
+    // Wrap the Rust hint implementation in a Box smart pointer inside a HintFunc
+    let hint = HintFunc(Box::new(print_a_hint));
+
+    //Instantiate the hint processor
+    let mut hint_processor = BuiltinHintProcessor::new_empty();
+
+    //Add the custom hint, together with the Python code
+    hint_processor.add_hint(String::from("print(ids.a)"), Rc::new(hint));
+
+    let file = File::open(Path::new(program_path)).expect("Couldn't load file");
+    let mut reader = BufReader::new(file);
+    let mut buffer = Vec::<u8>::new();
+    reader.read_to_end(&mut buffer).expect("Couldn't read file");
+
+    //Run the cairo program
+    let (_cairo_runner, virtual_machine) = cairo_run(
+        &buffer,
+        &CairoRunConfig {
+            layout: "all_cairo",
+            ..Default::default()
+        },
+        &mut hint_processor,
+    )
+    .expect("Couldn't run program");
+    assert_python_and_rust_output_match("build/different_output.json", virtual_machine);
+}