diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9ddc1e2fb..bfaf40a7a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,8 +78,6 @@ jobs: # It might be related to boxroot dependency, and we would need to bump # up the ocaml-rs dependency ocaml_version: ["4.14"] - os: ["ubuntu-latest"] - node: [20] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -96,7 +94,7 @@ jobs: with: rust_toolchain_version: ${{ matrix.rust_toolchain_version }} - - name: Rust Cache + - name: Apply the Rust smart cacheing uses: Swatinem/rust-cache@v2 - name: Use shared OCaml setting up steps diff --git a/.github/workflows/o1vm-ci.yml b/.github/workflows/o1vm-ci.yml new file mode 100644 index 0000000000..75763778ca --- /dev/null +++ b/.github/workflows/o1vm-ci.yml @@ -0,0 +1,120 @@ +name: o1vm CI + +on: + workflow_dispatch: + pull_request: + paths: + [ + "o1vm/**", + "folding/**", + "groupmap/**", + "kimchi/**", + "msm/**", + "curves/**", + "poseidon/**", + "poly-commitment/", + "internal-tracing/**", + "srs/**", + "utils/**", + ] + push: + branches: + - master + paths: + [ + "o1vm/**", + "folding/**", + "groupmap/**", + "kimchi/**", + "msm/**", + "curves/**", + "poseidon/**", + "poly-commitment/", + "internal-tracing/**", + "srs/**", + "utils/**", + ] + +env: + # https://doc.rust-lang.org/cargo/reference/profiles.html#release + # Disable for the time being since it fails with the "attempt to multiply with overflow" error. + # Known issue yeat to be fixed. + # RUSTFLAGS: -Coverflow-checks=y -Cdebug-assertions=y + # https://doc.rust-lang.org/cargo/reference/profiles.html#incremental + CARGO_INCREMENTAL: 1 + # https://nexte.st/book/pre-built-binaries.html#using-nextest-in-github-actions + CARGO_TERM_COLOR: always + # 30 MB of stack for Keccak tests + RUST_MIN_STACK: 31457280 + +jobs: + run_o1vm_with_cached_data: + name: Run o1vm with cached data + # We run only one of the matrix options on the toffee `hetzner-1` self-hosted GitHub runner. + # Only in this configuration we enable tests with the code coverage data gathering. + runs-on: ["ubuntu-latest"] + strategy: + matrix: + rust_toolchain_version: ["1.74"] + # FIXME: currently not available for 5.0.0. + # It might be related to boxroot dependency, and we would need to bump + # up the ocaml-rs dependency + ocaml_version: ["4.14"] + services: + o1vm-e2e-testing-cache: + image: o1labs/proof-systems:o1vm-e2e-testing-cache + volumes: + - /tmp:/tmp/cache + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Use shared Rust toolchain setting up steps + uses: ./.github/actions/toolchain-shared + with: + rust_toolchain_version: ${{ matrix.rust_toolchain_version }} + + - name: Apply the Rust smart cacheing + uses: Swatinem/rust-cache@v2 + + - name: Use shared OCaml setting up steps + uses: ./.github/actions/ocaml-shared + with: + ocaml_version: ${{ matrix.ocaml_version }} + + - name: Install the Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + check-latest: true + + - name: Install the Go + uses: actions/setup-go@v5 + with: + go-version: "1.21.0" + + - name: Install the Foundry + uses: foundry-rs/foundry-toolchain@v1 + + - name: Build the OP program + run: | + cd o1vm + make -C ./ethereum-optimism/op-program op-program + cd .. + + - name: Start the local HTTP server + run: | + python -m http.server 8765 & + + # + # Tests + # + + - name: Execute o1vm in E2E flavor using cached data + run: | + eval $(opam env) + cd o1vm + unzip -q -o /tmp/o1vm-e2e-testing-cache.zip -d ./ + RUN_WITH_CACHED_DATA="y" FILENAME="env-for-latest-l2-block.sh" O1VM_FLAVOR="pickles" STOP_AT="=3000000" ./run-code.sh diff --git a/o1vm/Dockerfile-e2e-testing-cache b/o1vm/Dockerfile-e2e-testing-cache new file mode 100644 index 0000000000..deeb1c29f6 --- /dev/null +++ b/o1vm/Dockerfile-e2e-testing-cache @@ -0,0 +1,12 @@ +FROM alpine + +# Add necessary utilities +RUN apk add --no-cache shadow unzip zip + +# Copy the files from the host to the Docker image +# Assuming the files and folder are located +# in the same directory as the current Dockerfile on the host +COPY o1vm-e2e-testing-cache.zip /tmp/cache-source/o1vm-e2e-testing-cache.zip + +# Command to copy the files to mounted volume +CMD ["/bin/sh", "-c", "cp -r /tmp/cache-source/* /tmp/cache/"] diff --git a/o1vm/README.md b/o1vm/README.md index 2644677e29..0a7cc6540b 100644 --- a/o1vm/README.md +++ b/o1vm/README.md @@ -1,4 +1,4 @@ -## o1VM: a zero-knowledge virtual machine +# o1VM: a zero-knowledge virtual machine This crate contains an implementation of different components used to build a zero-knowledge virtual machine. For now, the implementation is specialised for @@ -54,7 +54,7 @@ gvm install go1.21 gvm use go1.21s ``` -You also will need to install the [Foundry](https://getfoundry.sh/) toolkit +You also will need to install the [Foundry](https://getfoundry.sh/) toolkit in order to utilize applicaitons like `cast`. ```shell @@ -64,6 +64,7 @@ foundryup You will also need to install jq with your favorite packet manager. eg. on Ubuntu + ```shell sudo apt-get install jq ``` @@ -71,11 +72,13 @@ sudo apt-get install jq ## Running the Optimism demo Start by initializing the submodules: + ```bash git submodule init && git submodule update ``` Create an executable `rpcs.sh` file like: + ```bash #!/usr/bin/env bash export L1_RPC=http://xxxxxxxxx @@ -86,29 +89,34 @@ export L1_BEACON_RPC=http://xxxxxxxxx If you just want to test the state transition between the latest finalized L2 block and its predecessor: + ```bash ./run-code.sh ``` By default this will also create a script named `env-for-latest-l2-block.sh` with a snapshot of all the information that you need to rerun the same test again: + ```bash FILENAME=env-for-latest-l2-block.sh bash run-code.sh ``` Alternatively, you also have the option to test the state transition between a specific block and its predecessor: + ```bash # Set -n to the desired block transition you want to test. ./setenv-for-l2-block.sh -n 12826645 ``` In this case, you can run the demo using the following format: + ```bash FILENAME=env-for-l2-block-12826645.sh bash run-code.sh ``` In either case, `run-code.sh` will: + 1. Generate the initial state. 2. Execute the OP program. 3. Execute the OP program through the Cannon MIPS VM. @@ -117,6 +125,7 @@ In either case, `run-code.sh` will: ## Flavors Different versions/flavors of the o1vm are available. + - [legacy](./src/legacy/mod.rs) - to be deprecated. - [pickles](./src/pickles/mod.rs) (currently the default) @@ -126,6 +135,7 @@ environment variable `O1VM_FLAVOR`. ## Testing the preimage read Run: + ```bash ./test_preimage_read.sh [OP_DB_DIRECTORY] [NETWORK_NAME] ``` @@ -134,3 +144,34 @@ The default value for `OP_DB_DIRECTORY` would be the one from `setenv-for-latest-l2-block.sh` if the parameter is omitted. The `NETWORK_NAME` defaults to `sepolia`. + +## Running the o1vm with cached data + +If you want to run the o1vm with cached data, you can use the following steps: + +- Make sure you have [Docker Engine](https://docs.docker.com/engine/install/) and [Python3](https://www.python.org/downloads/) installed on your machine. +- Fetch the cached data by executing the following command (it might take some time): + +```shell +./fetch-e2e-testing-cache.sh +``` + +- Start the simple HTTP server (in background or in another terminal session): + +```shell +python3 -m http.server 8765 & +``` + +- Then run the o1vm with the following command: + +```shell +RUN_WITH_CACHED_DATA="y" FILENAME="env-for-latest-l2-block.sh" O1VM_FLAVOR="pickles" STOP_AT="=3000000" ./run-code.sh +``` + +- Don't forget to stop the HTTP server after you are done. + +- You can clean the cached data by executing the following command: + +```shell +./clear-e2e-testing-cache.sh +``` diff --git a/o1vm/clear-e2e-testing-cache.sh b/o1vm/clear-e2e-testing-cache.sh new file mode 100755 index 0000000000..8379737f49 --- /dev/null +++ b/o1vm/clear-e2e-testing-cache.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +START=$(date +%s) + +echo "" +echo "Cleaning up the E2E testing cache file system..." +echo "" +rm -rf op-program-db-for-latest-l2-block \ + env-for-latest-l2-block.sh \ + snapshot-state-3000000.json \ + state.json \ + meta.json \ + out.json \ + cpu.pprof + +RUNTIME=$(($(date +%s) - START)) +echo "" +echo "Execution time: ${RUNTIME} s" +echo "" diff --git a/o1vm/fetch-e2e-testing-cache.sh b/o1vm/fetch-e2e-testing-cache.sh new file mode 100755 index 0000000000..b090d80c86 --- /dev/null +++ b/o1vm/fetch-e2e-testing-cache.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +START=$(date +%s) + +echo "" +echo "Fetching the E2E testing cache..." +echo "" +docker run --rm -it --name o1vm-e2e-testing-cache --pull=always -v ./:/tmp/cache o1labs/proof-systems:o1vm-e2e-testing-cache +unzip -q -o o1vm-e2e-testing-cache.zip -d ./ +rm -rf o1vm-e2e-testing-cache.zip + +RUNTIME=$(($(date +%s) - START)) +echo "" +echo "Execution time: ${RUNTIME} s" +echo "" diff --git a/o1vm/publish-e2e-testing-cache.sh b/o1vm/publish-e2e-testing-cache.sh new file mode 100755 index 0000000000..397617fd19 --- /dev/null +++ b/o1vm/publish-e2e-testing-cache.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +START=$(date +%s) +# ARCH="amd64" +DOCKER_FILE_NAME="Dockerfile-e2e-testing-cache" +DOCKER_HUB_USER_NAME="o1labs" +DOCKER_HUB_REPO_NAME="proof-systems" +DOCKER_HUB_IMAGE_TAG="o1vm-e2e-testing-cache" +DOCKER_IMAGE_FULL_NAME="${DOCKER_HUB_USER_NAME}/${DOCKER_HUB_REPO_NAME}:${DOCKER_HUB_IMAGE_TAG}" + +echo "" +echo "Preparing the file system..." +echo "" +zip -r -q o1vm-e2e-testing-cache.zip op-program-db-for-latest-l2-block \ + env-for-latest-l2-block.sh \ + snapshot-state-3000000.json \ + state.json \ + meta.json \ + out.json \ + cpu.pprof + +echo "" +echo "Building the '${DOCKER_IMAGE_FULL_NAME}' Docker image..." +echo "" +docker rmi -f ${DOCKER_IMAGE_FULL_NAME} || true +docker rmi -f ${DOCKER_HUB_IMAGE_TAG} || true +# docker build --platform linux/${ARCH} -t ${DOCKER_HUB_IMAGE_TAG} -f ${DOCKER_FILE_NAME} . +docker build -t ${DOCKER_HUB_IMAGE_TAG} -f ${DOCKER_FILE_NAME} . + +echo "" +echo "Publishing the '${DOCKER_IMAGE_FULL_NAME}' Docker image..." +echo "" +docker tag ${DOCKER_HUB_IMAGE_TAG} ${DOCKER_IMAGE_FULL_NAME} +docker push ${DOCKER_IMAGE_FULL_NAME} +echo "" + +echo "" +echo "Cleaning up the file system..." +echo "" +rm -rf o1vm-e2e-testing-cache.zip + +RUNTIME=$(($(date +%s) - START)) +echo "" +echo "Execution time: ${RUNTIME} s" +echo "" diff --git a/o1vm/run-code.sh b/o1vm/run-code.sh index f07e325917..8753acf2c5 100755 --- a/o1vm/run-code.sh +++ b/o1vm/run-code.sh @@ -10,6 +10,10 @@ set -u source $FILENAME -./run-op-program.sh -./run-cannon.sh +if [ "${RUN_WITH_CACHED_DATA:-}" == "y" ]; then + echo "The Op-Program and the Cannon apps were not executed because the cached data usage was requested" +else + ./run-op-program.sh + ./run-cannon.sh +fi ./run-vm.sh diff --git a/o1vm/run-vm.sh b/o1vm/run-vm.sh index 0e18c416f0..34e61f4da7 100755 --- a/o1vm/run-vm.sh +++ b/o1vm/run-vm.sh @@ -4,40 +4,49 @@ set -euo pipefail O1VM_FLAVOR="${O1VM_FLAVOR:-pickles}" case ${O1VM_FLAVOR} in - "legacy") - BINARY_FLAVOR="legacy_o1vm" - ;; - "pickles") - BINARY_FLAVOR="pickles_o1vm" - ;; - *) - echo "${BASH_SOURCE[0]}:${LINENO}: only flavors 'legacy' and \ +"legacy") + BINARY_FLAVOR="legacy_o1vm" + ;; +"pickles") + BINARY_FLAVOR="pickles_o1vm" + ;; +*) + echo "${BASH_SOURCE[0]}:${LINENO}: only flavors 'legacy' and \ 'pickles' (default) are supported for now. Use the environment variable \ -O1VM_FLAVOR to one of these values to run the flavor you would like"; - exit 1 - ;; - esac +O1VM_FLAVOR to one of these values to run the flavor you would like" + exit 1 + ;; +esac + +if [ "${RUN_WITH_CACHED_DATA:-}" == "y" ]; then + echo "Setting the environment variables to use the cached data" + export OP_PROGRAM_DATA_DIR="$(pwd)/op-program-db-for-latest-l2-block" + export ZKVM_STATE_FILENAME="$(pwd)/snapshot-state-3000000.json" + # We need to set the L1 and L2 RPC endpoints for op-program to run successfully + export L1_RPC="http://localhost:8765" + export L2_RPC="http://localhost:8765" +fi cargo run --bin ${BINARY_FLAVOR} \ --all-features \ --release \ -p o1vm -- \ - --pprof.cpu \ - --info-at "${INFO_AT:-%10000000}" \ - --snapshot-state-at "${SNAPSHOT_STATE_AT:-%10000000}" \ - --proof-at never \ - --stop-at "${STOP_AT:-never}" \ - --input "${ZKVM_STATE_FILENAME:-./state.json}" \ - -- \ - ./ethereum-optimism/op-program/bin/op-program \ - --log.level DEBUG \ - --l1 "${L1_RPC}" \ - --l2 "${L2_RPC}" \ - --network sepolia \ - --datadir "${OP_PROGRAM_DATA_DIR}" \ - --l1.head "${L1_HEAD}" \ - --l2.head "${L2_HEAD}" \ - --l2.outputroot "${STARTING_OUTPUT_ROOT}" \ - --l2.claim "${L2_CLAIM}" \ - --l2.blocknumber "${L2_BLOCK_NUMBER}" \ - --server + --pprof.cpu \ + --info-at "${INFO_AT:-%10000000}" \ + --snapshot-state-at "${SNAPSHOT_STATE_AT:-%10000000}" \ + --proof-at never \ + --stop-at "${STOP_AT:-never}" \ + --input "${ZKVM_STATE_FILENAME:-./state.json}" \ + -- \ + ./ethereum-optimism/op-program/bin/op-program \ + --log.level DEBUG \ + --l1 "${L1_RPC}" \ + --l2 "${L2_RPC}" \ + --network sepolia \ + --datadir "${OP_PROGRAM_DATA_DIR}" \ + --l1.head "${L1_HEAD}" \ + --l2.head "${L2_HEAD}" \ + --l2.outputroot "${STARTING_OUTPUT_ROOT}" \ + --l2.claim "${L2_CLAIM}" \ + --l2.blocknumber "${L2_BLOCK_NUMBER}" \ + --server diff --git a/o1vm/src/cannon_cli.rs b/o1vm/src/cannon_cli.rs index 0501948fa1..063ec84a6e 100644 --- a/o1vm/src/cannon_cli.rs +++ b/o1vm/src/cannon_cli.rs @@ -66,9 +66,7 @@ pub fn main_cli() -> clap::Command { pub fn read_configuration(cli: &clap::ArgMatches) -> VmConfiguration { let input_state_file = cli.get_one::("input").unwrap(); - let output_state_file = cli.get_one::("output").unwrap(); - let metadata_file = cli.get_one::("meta").unwrap(); let proof_at = cli.get_one::("proof-at").unwrap(); diff --git a/o1vm/src/interpreters/mips/witness.rs b/o1vm/src/interpreters/mips/witness.rs index e3838832e4..74eb926997 100644 --- a/o1vm/src/interpreters/mips/witness.rs +++ b/o1vm/src/interpreters/mips/witness.rs @@ -1287,6 +1287,7 @@ impl Env { // Approximate instruction per seconds let how_many_steps = step as usize - start.step; + let ips = how_many_steps as f64 / elapsed.as_secs() as f64; let pages = self.memory.len(); diff --git a/o1vm/src/lookups.rs b/o1vm/src/lookups.rs index 1e55190fd2..eff69771f3 100644 --- a/o1vm/src/lookups.rs +++ b/o1vm/src/lookups.rs @@ -62,14 +62,15 @@ impl LookupTableID for LookupTableIDs { match value { 0 => PadLookup, 1 => RoundConstantsLookup, - 2 => ByteLookup, - 3 => RangeCheck16Lookup, - 4 => SparseLookup, - 5 => ResetLookup, - 6 => MemoryLookup, - 7 => RegisterLookup, - 8 => SyscallLookup, - 9 => KeccakStepLookup, + 2 => AtMost4Lookup, + 3 => ByteLookup, + 4 => RangeCheck16Lookup, + 5 => SparseLookup, + 6 => ResetLookup, + 7 => MemoryLookup, + 8 => RegisterLookup, + 9 => SyscallLookup, + 10 => KeccakStepLookup, _ => panic!("Invalid table ID"), } }