Add cairo native (#943)

* Added cairo native flag * Added cairo_native as dependency * Partial progress * Progress * Partial progress * Point to newly created branch on cairo native * Use updated version of cairo native and make test pass * Run test with storage_write and storage_read with cairo native * Tidy up code * Start unhardcoding stuff * Handle return values * Tidy up code a little * Added sierra programs cache * Add impl for emit_event and send_l1_message * Fix tests * Install LLVM on CI * Test * Test CI * Fix lint step * Save work in progress * Unhardcode calldata and entry point * Unhardcode more stuff * Fix test * Add basic implementation for call_contract * Add call to get_name to integration test * Make call_contract impl compile * Fix test * Pass the correct amount of builtins for every function * Improve test legibility * Write call_contract test skeleton * Finish writing test, still bugs to fix * implement get_execution_info, make increase_allowance work * More debugging * Basic test for call_contract working * More progress on testing ERC20 * More progress on test * Add caller and callee contracts * Fix call_contract test after merge * Fix callee address in tests * Polish some details * Remove use_cairo_native from TransactionExecutionContext * Write test skeleton * Add test contracts * Fix tests * Readd deleted contracts * Echo contract test passing * Update cairo compiler to version 2.2.0 * Calling another contract with events test is passing * Fix failing tests * Remove internal_calls field TODO in CallInfo returned by native_execute * Add event_emitter contract * Add cairo vm execution to the erc20 test for comparison * Add simple implementation for panics in native execution * Add some documentation in code * Assert equality between events, accessed_storage_keys and l1_l2 messages between native and vm runs * Add event_emitter contract * Remove print * Remove comments * Move native syscall handler to its own file * Add felt decode to string when program panics * Make cairo native an optional dependency behind a feature * Move execution result to cairo native * Add a README section explaining how to setup cairo native * Fix some clippy issues * Fix test compilation * CI test * Revert "CI test" This reverts commit 4631e5e. * CI test * Test * Test * Address comment about multiple cfgs * Remove unnecessary clone * Test * Test * Switch to special workflow for native integration tests * Fix workflow * Fix stuff after merge. * Fix clippy warnings. * Fix after merge. * Fix comments. * Fix `Makefile`. * Remove unused import. * Use transactional state. * update cairo native to llvm 17 and remove nightly requirement * update ci * upd ci * try to fix ci * use ubuntu on native * try to fix ci * not needed? * fix ci * update cairo native * fix nightly usage * try to fix ci * dont need a transactional state reader, simply clone the state reader * try to fix ci * format * fix again * fix if * values * force rebuild * make cache track cairo 2 version to trigger rebuilds * try no restore key * update readme * fix tests on ci * update cairo native commit * fix more tests on ci * fix cairo native interface * try ci without cache * setup rustup home * make param passing to cairo native not obscure * try * try again * remove large packages * remove large dirs * remove android too * polly is needed * needs sudo * fix cov * fix test for now * format * cleanup ci file * use pyenv if available * nightly not needed in readme * add .sierra as generated to gitattributes * fix gitattributes * add casm too --------- Co-authored-by: Javier Chatruc <[email protected]> Co-authored-by: Mariano Nicolini <[email protected]> Co-authored-by: Esteve Soler Arderiu <[email protected]> Co-authored-by: Edgar Luque <[email protected]>
lambdaclass · Oct 17, 2023 · 8ed7fb7 · 8ed7fb7
1 parent 484acb3
commit 8ed7fb7
Show file tree

Hide file tree

Showing 61 changed files with 15,183 additions and 659 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+*.sierra linguist-generated
+*.casm linguist-generated
diff --git a/.github/workflows/rust-tests.yml b/.github/workflows/rust-tests.yml
@@ -9,240 +9,139 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  RUST_TOOLCHAIN: 1.70.0
-  CAIRO_PROGRAMS_PATH: |
-    cairo_programs/**/*.casm
-    cairo_programs/**/*.sierra
-    cairo_programs/**/*.json
-    starknet_programs/**/*.casm
-    starknet_programs/**/*.sierra
-    starknet_programs/**/*.json
-    !starknet_programs/raw_contract_classes/*
+  RUST_TOOLCHAIN: '1.72.1'
 
 jobs:
-  build-programs:
-    strategy:
-      matrix:
-        program-target: [
-          compile-cairo,
-          compile-starknet,
-          compile-cairo-1-casm,
-          compile-cairo-1-sierra,
-          compile-cairo-2-casm,
-          compile-cairo-2-sierra,
-        ]
-    name: Build Cairo programs
-    runs-on: ubuntu-22.04
+  build:
+    name: Build with release profile
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+    - name: Install Rust $RUST_TOOLCHAIN
+      uses: dtolnay/rust-toolchain@master
       with:
-        fetch-depth: 0
-
-    - name: Fetch from cache
-      uses: actions/cache@v3
-      id: cache-programs
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        restore-keys: ${{ matrix.program-target }}-cache-
-
-    # This is not pretty, but we need `make` to see the compiled programs are
-    # actually newer than the sources, otherwise it will try to rebuild them
-    - name: Restore timestamps
-      uses: chetan/git-restore-mtime-action@v1
-
+        toolchain: $RUST_TOOLCHAIN
+        components: rustfmt, clippy
     - name: Python3 Build
-      if: ${{ steps.cache-programs.outputs.cache-hit != 'true' }}
       uses: actions/setup-python@v4
       with:
         python-version: '3.9'
         cache: 'pip'
-
-    - name: Install deps
-      if: ${{ steps.cache-programs.outputs.cache-hit != 'true' }}
-      run: make deps
-
-    - name: Build programs
-      if: ${{ steps.cache-programs.outputs.cache-hit != 'true' }}
-      run: make -j ${{ matrix.program-target }}
-
-  # NOTE: used to reduce the amount of cache steps we need in later jobs
-  # TODO: remove this cache once the workflow finishes
-  merge-caches:
-    name: Merge Cairo programs cache
-    runs-on: ubuntu-22.04
-    needs: build-programs
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Fetch from cache (compile-cairo)
-      uses: actions/cache/restore@v3
-      id: cache-programs
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-cairo-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Fetch from cache (compile-starknet)
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-starknet-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Fetch from cache (compile-cairo-1-casm)
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-cairo-1-casm-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Fetch from cache (compile-cairo-1-sierra)
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-cairo-1-sierra-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Fetch from cache (compile-cairo-2-casm)
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-cairo-2-casm-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Fetch from cache (compile-cairo-2-sierra)
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-cairo-2-sierra-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Fetch from cache (compile-cairo-2-sierra)
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: compile-cairo-2-sierra-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
-    - name: Merge caches
-      uses: actions/cache/save@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-
-  build:
-    name: Build with release profile
-    needs: merge-caches
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@stable
-      with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          components: rustfmt, clippy
     - uses: Swatinem/rust-cache@v2
       with:
         cache-on-failure: true
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Fetch programs
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
 
+    - name: Install deps
+      run: make deps
     - name: Build
-      run: cargo build --release --workspace
+      run: make build
 
   lint:
     name: Lint with fmt and clippy
-    needs: merge-caches
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@stable
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install Rust $RUST_TOOLCHAIN
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: $RUST_TOOLCHAIN
+        components: rustfmt, clippy
+    - name: Python3 Build
+      uses: actions/setup-python@v4
       with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          components: rustfmt, clippy
+        python-version: '3.9'
+        cache: 'pip'
     - uses: Swatinem/rust-cache@v2
       with:
         cache-on-failure: true
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Fetch programs
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
+    - name: Install deps
+      run: make deps
     - name: Format
       run: cargo fmt --all -- --check
     - name: Run clippy
-      run: cargo clippy --workspace --all-targets -- -D warnings
+      run: make clippy
 
   tests:
     env:
       INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
+      MLIR_SYS_170_PREFIX: /usr/lib/llvm-17/
+      TABLEGEN_170_PREFIX: /usr/lib/llvm-17/
     strategy:
       fail-fast: false
       matrix:
-        target: [ test-cairo-1, test-cairo-2, test-doctests ]
+        target: [ test-cairo-1, test-cairo-2, test-doctests, test-cairo-native ]
     name: Run tests
-    needs: merge-caches
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@stable
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install Rust $RUST_TOOLCHAIN
+      uses: dtolnay/rust-toolchain@master
       with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          components: rustfmt, clippy
+        toolchain: $RUST_TOOLCHAIN
+        components: rustfmt, clippy
+    - name: Python3 Build
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        cache: 'pip'
     - uses: Swatinem/rust-cache@v2
       with:
         cache-on-failure: true
-    - name: Checkout
-      uses: actions/checkout@v3
 
-    - name: Fetch programs
-      uses: actions/cache/restore@v3
-      with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
-
+    - name: Install deps
+      run: make deps
+
     - name: Install testing tools
       # TODO: remove `if` when nextest adds doctests support
       if: ${{ matrix.target != 'test-doctests' }}
       uses: taiki-e/install-action@v2
       with:
         tool: [email protected]
 
+    - name: check and free hdd space left
+      run: |
+        echo "Listing 20 largest packages"
+        dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
+        df -h
+        sudo apt-get update
+        sudo apt-get remove -y '^llvm-.*'
+        sudo apt-get remove -y 'php.*'
+        sudo apt-get remove -y '^dotnet-.*'
+        sudo apt-get remove -y '^temurin-.*'
+        sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox powershell mono-devel
+        sudo apt-get autoremove -y
+        sudo apt-get clean
+        df -h
+        echo "Removing large directories"
+        # deleting 15GB
+        sudo rm -rf /usr/share/dotnet/
+        sudo rm -rf /usr/local/lib/android
+        df -h
+    - name: add llvm deb repository
+      uses: myci-actions/add-deb-repo@10
+      with:
+        repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
+        repo-name: llvm-repo
+        keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
+    - name: Install LLVM
+      run: sudo apt-get install llvm-17 llvm-17-dev llvm-17-runtime clang-17 clang-tools-17 lld-17 libpolly-17-dev libmlir-17-dev mlir-17-tools
     - name: Run tests (${{ matrix.target }})
       run: make ${{ matrix.target }}
 
   coverage:
-    needs: merge-caches
     name: Generate and upload coverage report
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@nightly
-      with:
-        toolchain: ${{ env.RUST_TOOLCHAIN }}
-
-    - name: Set nightly as default
-      run: rustup default nightly
 
-    - name: Install testing tools
-      uses: taiki-e/install-action@v2
+    - name: Install Rust nightly
+      uses: dtolnay/rust-toolchain@master
       with:
-        tool: [email protected],cargo-llvm-cov
+        toolchain: nightly
 
     - uses: Swatinem/rust-cache@v2
       with:
@@ -255,13 +154,17 @@ jobs:
         path: lcov.info
         key: coverage-cache-${{ github.sha }}
 
-    - name: Fetch programs
-      if: steps.restore-report.outputs.cache-hit != 'true'
-      uses: actions/cache/restore@v3
+    - name: Python3 Build
+      uses: actions/setup-python@v4
       with:
-        path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'starknet_programs/**/*.cairo') }}
-        fail-on-cache-miss: true
+        python-version: '3.9'
+        cache: 'pip'
+    - uses: Swatinem/rust-cache@v2
+      with:
+        cache-on-failure: true
+
+    - name: Install deps
+      run: make deps
 
     - name: Generate coverage report
       if: steps.restore-report.outputs.cache-hit != 'true'