feat: prepare for ontake upgrade (#329)

* prepare for ontake upgrade

Signed-off-by: smtmfft <[email protected]>

* update Cargo.toml

Signed-off-by: smtmfft <[email protected]>

* use forked verifier set together with taiko hard forks

Signed-off-by: smtmfft <[email protected]>

* fix hekla path test

* sync with protocol change

* sync latest protocol

* rebase main

* use defaulte l1_inc_block as anchor-1

* refactor(core): [WIP] inital preflight refactor

* refactor(core): [WIP] continue preflight refactor

* fix(host,tasks): add `l1_inclusive_block_number` field

* chore(lib,sgx): run cargo fmt

* fix(lib): clippy fixes

* chore(core): fix lints

* fix(core): fix block conversion for parent block

* refactor(core): move preflight to mod and separate into util

* feat(host): [WIP] add integration test skeleton

* switch back to origin reth branch

* feat(host): [WIP] improve test description and options

* chore(host): hide integration test behind integration feature

* feat(host): [WIP] add cancellation and retry to the test flow

* chore(ci): [WIP] add integration tests for native

* fix(host): remove comment from feature flag

* feat(host): add test for v1, prunning and reporting

* feat(host): support custom config path via env vars for testing

* fix(ci): add env var for config path for integration testing

* fix(host): fix clippy warning

* fix(ci): remove `@`

* feat(host): separate scenarios and run sequentially

* fix(host): fix typo in functionality

* chore(host): add comment

* refactor(core): use fork instead of inclusion block

* feat(ci): add support for testing different proof types

* fix(host): pass string instead of type

* fix(script): fix typo

* chore(core): add detail to transaction execution failure

* refactor(lib): extract metadata conversion

* refactor(lib): clean up preflight utils

* fix(core): fix clippy issue

* feat(ci): add integration tests for all provers

* fix(ci): remove enable feature

* fix(ci): add prover elf building

* refactor(host): serialize into status error instead of manual

* refactor(host): clean up error conversion

* refactor(core): rename l1 inclusion block number field

* fix(core,host,tasks): save files after rename

* feat(core): temporarily use new reth ontake branch (#357)

* temporarily use new reth ontake branch

* Update core/src/preflight/util.rs

Co-authored-by: Petar Vujović <[email protected]>

* chore(lib): run fmt

---------

Co-authored-by: Petar Vujovic <[email protected]>

* chore(core,ci): renaming

* update pre-compiled image

Signed-off-by: smtmfft <[email protected]>

* update prebuilt image

Signed-off-by: smtmfft <[email protected]>

* test prebuilt images

Signed-off-by: qcloud <[email protected]>

* debg CI (#359)

* debg CI

Signed-off-by: smtmfft <[email protected]>

* switch reth back to v1.0.0-rc.2-taiko branch

Signed-off-by: smtmfft <[email protected]>

---------

Signed-off-by: smtmfft <[email protected]>

* chore(ci): use shorter flag passing

* fix(lib): support non taiko forked chains for block proving

* fix(ci): building provers for tests (#360)

* fix CI

* fix ci

Signed-off-by: smtmfft <[email protected]>

* fix ci

Signed-off-by: smtmfft <[email protected]>

* fix ci

* fix ci

Signed-off-by: smtmfft <[email protected]>

* fix ci

Signed-off-by: smtmfft <[email protected]>

* fix ci

Signed-off-by: smtmfft <[email protected]>

* chore(host): add more descriptive asserts

* fix(host): use blob proof type other than none

* fix(host): add `to_string` call

* revert install.sh

Signed-off-by: smtmfft <[email protected]>

* chore(ci): remove integration from CI until cwd fix

* revert ci changes

Signed-off-by: smtmfft <[email protected]>

---------

Signed-off-by: smtmfft <[email protected]>
Co-authored-by: Petar Vujovic <[email protected]>

* fix(ci): remove comments which broke CI

* fix(ci): add single step

* refine pipeline stdout message

Signed-off-by: smtmfft <[email protected]>

* update to latest ontake taiko-reth

Signed-off-by: smtmfft <[email protected]>

* add missing binary

Signed-off-by: smtmfft <[email protected]>

* fixing r0 ci (#362)

Signed-off-by: smtmfft <[email protected]>

* fix ci

Signed-off-by: smtmfft <[email protected]>

* fix ci

Signed-off-by: smtmfft <[email protected]>

* add ci status report script

* update to fixed hekla ontake height

Signed-off-by: smtmfft <[email protected]>

---------

Signed-off-by: smtmfft <[email protected]>
Signed-off-by: qcloud <[email protected]>
Co-authored-by: Petar Vujovic <[email protected]>

.github/workflows/ci-integration-reusable.yml

@@ -0,0 +1,52 @@
+name: CI Integration test - Reusable
+
+on:
+  workflow_call:
+    inputs:
+      version_name:
+        type: string
+        required: true
+      version_toolchain:
+        type: string
+        required: true
+
+env:
+  CI: 1
+  CARGO_TERM_COLOR: always
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  build-run-integration:
+    name: Build and run integration
+    runs-on: [taiko-runner]
+    timeout-minutes: 120
+
+    env:
+      TARGET: ${{ inputs.version_name }}
+      CI: 1
+      MOCK: 1
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ inputs.version_toolchain }}
+          profile: minimal
+
+      - name: Setup sccache
+        if: ${{ inputs.version_name }} == risc0
+        uses: risc0/risc0/.github/actions/[email protected]
+
+      - name: Install ${{ inputs.version_name }}
+        run: make install
+
+      - name: Build ${{ inputs.version_name }} prover
+        run: make build
+
+    #  TODO:(petar) re add to CI after fixing sgx directory setup
+    #
+    #   - name: Run integration test for ${{ inputs.version_name }} prover
+    #     run: make integration

.github/workflows/ci-native.yml

@@ -9,3 +9,10 @@ jobs:
     with:
       version_name: "native"
       version_toolchain: "nightly-2024-04-17"
+
+  integration-test-native:
+    name: Run integration tests on native
+    uses: ./.github/workflows/ci-integration-reusable.yml
+    with:
+      version_name: "native"
+      version_toolchain: "nightly-2024-04-17"

.github/workflows/ci-risc0.yml

@@ -18,3 +18,10 @@ jobs:
     with:
       version_name: "risc0"
       version_toolchain: "stable"
+
+  integration-test-risc0:
+    name: Run integration tests on risc0
+    uses: ./.github/workflows/ci-integration-reusable.yml
+    with:
+      version_name: "risc0"
+      version_toolchain: "nightly-2024-04-17"

.github/workflows/ci-sgx-all.yml

@@ -12,17 +12,24 @@ on:
   merge_group:
 
 jobs:
-    build-test-sgx:
-      name: Build and test sgx
-      uses: ./.github/workflows/ci-build-test-reusable.yml
-      with:
-        version_name: "sgx"
-        version_toolchain: "stable"
+  build-test-sgx:
+    name: Build and test sgx
+    uses: ./.github/workflows/ci-build-test-reusable.yml
+    with:
+      version_name: "sgx"
+      version_toolchain: "stable"
 
-    build-test-sgx-docker:
-      name: Build and test sgx with Docker
-      uses: ./.github/workflows/ci-sgx-docker.yml
+  integration-test-sgx:
+    name: Run integration tests on sgx
+    uses: ./.github/workflows/ci-integration-reusable.yml
+    with:
+      version_name: "sgx"
+      version_toolchain: "nightly-2024-04-17"
 
-    build-test-sgx-hardware:
-      name: Build and test sgx in hardware
-      uses: ./.github/workflows/ci-sgx-hardware.yml
+  build-test-sgx-docker:
+    name: Build and test sgx with Docker
+    uses: ./.github/workflows/ci-sgx-docker.yml
+
+  build-test-sgx-hardware:
+    name: Build and test sgx in hardware
+    uses: ./.github/workflows/ci-sgx-hardware.yml

.github/workflows/ci-sp1.yml

@@ -18,3 +18,10 @@ jobs:
     with:
       version_name: "sp1"
       version_toolchain: "nightly-2024-04-18"
+
+  integration-test-sp1:
+    name: Run integration tests on sp1
+    uses: ./.github/workflows/ci-integration-reusable.yml
+    with:
+      version_name: "sp1"
+      version_toolchain: "nightly-2024-04-17"