From 949b027a6594b779973c002fa13ff2f33d8e22d7 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:15:56 +0000 Subject: [PATCH 01/17] ci: move `build-rust` to runs-on Further, this commit enables S3 caching via https://runs-on.com/caching/s3-cache-for-github-actions. --- .github/workflows/ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba1ecc7576..c7bc4a5f36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,13 @@ jobs: build-rust: name: "Rust build (${{matrix.config.name}})" - runs-on: ubuntu-latest + runs-on: + - runs-on=${{ github.run_id }} + - family=c7i.8xlarge + - image=ubuntu24-full-x64 + - spot=false + - tag=${{ matrix.id }} + env: # disable lints for build, they will be caught in Rust lint job. RUSTFLAGS: '-A warnings' @@ -142,10 +148,10 @@ jobs: rustflags: 'RUSTFLAGS="-A warnings --cfg getrandom_backend=\"wasm_js\""' args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-tui" steps: + - uses: runs-on/action@v1 + - uses: runs-on/cache@v4 - uses: actions/checkout@v4 - - uses: ./.github/actions/cleanup - uses: rui314/setup-mold@v1 - - uses: ./.github/actions/setup-rust with: targets: ${{matrix.config.target || ''}} - uses: ./.github/actions/setup-c++ From 16a04973fa8564cabbd406bd0d97f440c486584f Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:26:36 +0000 Subject: [PATCH 02/17] define cache key --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7bc4a5f36..1334bcc81e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,8 @@ jobs: steps: - uses: runs-on/action@v1 - uses: runs-on/cache@v4 + with: + key: ${{ runner.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions/checkout@v4 - uses: rui314/setup-mold@v1 with: From d369bc615687ce7b1e9d87f417ce0a700e07b083 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:31:39 +0000 Subject: [PATCH 03/17] define path --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1334bcc81e..b232516781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,7 @@ jobs: - uses: runs-on/action@v1 - uses: runs-on/cache@v4 with: + path: . key: ${{ runner.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions/checkout@v4 - uses: rui314/setup-mold@v1 From d8b87eefdef64a671dd2ab2c3a2faaa2806e7203 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:41:15 +0000 Subject: [PATCH 04/17] disable the cache for now --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b232516781..06a37f9681 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,7 @@ jobs: - runs-on=${{ github.run_id }} - family=c7i.8xlarge - image=ubuntu24-full-x64 + - disk=large - spot=false - tag=${{ matrix.id }} @@ -149,10 +150,10 @@ jobs: args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-tui" steps: - uses: runs-on/action@v1 - - uses: runs-on/cache@v4 - with: - path: . - key: ${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + # - uses: runs-on/cache@v4 + # with: + # path: . + # key: ${{ runner.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions/checkout@v4 - uses: rui314/setup-mold@v1 with: From a2d23f3144d070e7d105909c8872b15e88a4d67f Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:49:06 +0000 Subject: [PATCH 05/17] install wasm target --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06a37f9681..9baf4d214c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,9 @@ jobs: with: targets: ${{matrix.config.target || ''}} - uses: ./.github/actions/setup-c++ + - name: Install wasm32 target + if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} + run: rustup target add wasm32-unknown-unknown - name: Install cargo-hack uses: taiki-e/install-action@cargo-hack - name: Rust Build (${{matrix.config.name}}) From 16ed521a7666eb99d5b02d2fc5a706b50cb5bd63 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:51:52 +0000 Subject: [PATCH 06/17] disable s3 cache for now --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9baf4d214c..88dd2eeb3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,10 +150,6 @@ jobs: args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-tui" steps: - uses: runs-on/action@v1 - # - uses: runs-on/cache@v4 - # with: - # path: . - # key: ${{ runner.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions/checkout@v4 - uses: rui314/setup-mold@v1 with: From c6cbbeb758220f5c364e4ab8a9ab27bac809789e Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 11:59:55 +0000 Subject: [PATCH 07/17] fix tag --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88dd2eeb3f..f389068e76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - spot=false - - tag=${{ matrix.id }} + - tag=${{ matrix.name }} env: # disable lints for build, they will be caught in Rust lint job. From 6e0380e26798d05c87082f053b1a9014e18297aa Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 12:04:59 +0000 Subject: [PATCH 08/17] fix tag --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f389068e76..16d7bdd259 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - spot=false - - tag=${{ matrix.name }} + - tag=${{ matrix.config.name }} env: # disable lints for build, they will be caught in Rust lint job. From 35e761e612a2152f88596fa0e9816a93b0bdbfa7 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 12:12:47 +0000 Subject: [PATCH 09/17] fix all tags --- .github/workflows/bench-pr.yml | 2 +- .github/workflows/bench.yml | 2 +- .github/workflows/sql-benchmarks.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 24c653e217..df8363446d 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -30,7 +30,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - spot=false - - tag=${{ matrix.id }} + - tag=${{ matrix.benchmark.id }} strategy: matrix: benchmark: diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 782bfbc1e4..d4dd85ff8a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -36,7 +36,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - spot=false - - tag=${{ matrix.id }} + - tag=${{ matrix.benchmark.id }} strategy: matrix: benchmark: diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 7251e6100c..d5f9cbff06 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -31,7 +31,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - spot=false - - tag=${{ matrix.id }} + - tag=${{ matrix.include.id }} steps: - uses: runs-on/action@v1 From 6e5c5a3288415952d6eec9fec772a48fe0249338 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 12:26:32 +0000 Subject: [PATCH 10/17] move more tasks to aws runners --- .github/workflows/ci.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16d7bdd259..0383cb73bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,12 +165,17 @@ jobs: check-min-deps: name: "Check build with minimal dependencies" - runs-on: ubuntu-latest + runs-on: + - runs-on=${{ github.run_id }} + - family=c7i.8xlarge + - image=ubuntu24-full-x64 + - disk=large + - spot=false + - tag=rust-min-deps steps: + - uses: runs-on/action@v1 - uses: actions/checkout@v4 - - uses: ./.github/actions/cleanup - uses: rui314/setup-mold@v1 - - uses: ./.github/actions/setup-rust - uses: ./.github/actions/setup-c++ - uses: taiki-e/install-action@cargo-hack - uses: taiki-e/install-action@cargo-minimal-versions @@ -178,12 +183,17 @@ jobs: rust-lint: name: "Rust (lint)" - runs-on: ubuntu-latest + runs-on: + - runs-on=${{ github.run_id }} + - family=c7i.8xlarge + - image=ubuntu24-full-x64 + - disk=large + - spot=false + - tag=rust-lint steps: + - uses: runs-on/action@v1 - uses: actions/checkout@v4 - - uses: ./.github/actions/cleanup - uses: rui314/setup-mold@v1 - - uses: ./.github/actions/setup-rust - uses: ./.github/actions/setup-c++ - name: Rust Lint - Format run: cargo fmt --all --check From 832d6c190480ebe201ad19dbbb447e8dcd60b6f0 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 14:38:13 +0000 Subject: [PATCH 11/17] retry variety --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0383cb73bd..4d03b2b411 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,7 +114,8 @@ jobs: name: "Rust build (${{matrix.config.name}})" runs-on: - runs-on=${{ github.run_id }} - - family=c7i.8xlarge + - family=c7i + - cpu: [4, 6, 8] - image=ubuntu24-full-x64 - disk=large - spot=false From 6793ef807a8927dabcc86ca6f79670b7e3c45683 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 14:42:17 +0000 Subject: [PATCH 12/17] families --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d03b2b411..78c5cca049 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,11 +114,10 @@ jobs: name: "Rust build (${{matrix.config.name}})" runs-on: - runs-on=${{ github.run_id }} - - family=c7i - - cpu: [4, 6, 8] + - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - cpu=8 - image=ubuntu24-full-x64 - disk=large - - spot=false - tag=${{ matrix.config.name }} env: From 0634d0bfcc8421647d7234dfa2ca78f04ed3d0cd Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 15:00:22 +0000 Subject: [PATCH 13/17] more variety --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78c5cca049..ed522baa9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,8 @@ jobs: name: "Check build with minimal dependencies" runs-on: - runs-on=${{ github.run_id }} - - family=c7i.8xlarge + - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - cpu=8 - image=ubuntu24-full-x64 - disk=large - spot=false @@ -185,7 +186,8 @@ jobs: name: "Rust (lint)" runs-on: - runs-on=${{ github.run_id }} - - family=c7i.8xlarge + - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - cpu=8 - image=ubuntu24-full-x64 - disk=large - spot=false From 9508d08304ef2f5b4e160d6cc61b2b715f60eb8d Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 15:03:05 +0000 Subject: [PATCH 14/17] reenable spot instances --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed522baa9e..ae545966b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,7 +171,6 @@ jobs: - cpu=8 - image=ubuntu24-full-x64 - disk=large - - spot=false - tag=rust-min-deps steps: - uses: runs-on/action@v1 @@ -190,7 +189,6 @@ jobs: - cpu=8 - image=ubuntu24-full-x64 - disk=large - - spot=false - tag=rust-lint steps: - uses: runs-on/action@v1 @@ -257,9 +255,9 @@ jobs: name: Benchmark with Codspeed runs-on: - runs-on=${{ github.run_id }} - - family=c7i.8xlarge + - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - cpu=8 - image=ubuntu24-full-x64 - - spot=false steps: - uses: runs-on/action@v1 From 7c400d17ba456c8c303871ee15340fb6eeca1b12 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 15:09:07 +0000 Subject: [PATCH 15/17] fixed codspeed instance --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae545966b7..687c87076d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,8 +255,7 @@ jobs: name: Benchmark with Codspeed runs-on: - runs-on=${{ github.run_id }} - - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a - - cpu=8 + - family=c7i.8xlarge - image=ubuntu24-full-x64 steps: From 6e197bec672789124d4c7b9e977d280aada526bf Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 15:18:40 +0000 Subject: [PATCH 16/17] disable spot again --- .github/workflows/ci.yml | 4 ++++ .github/workflows/fuzz.yml | 2 ++ .github/workflows/generate-tpch-s3.yml | 1 + 3 files changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 687c87076d..747f817b05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,7 @@ jobs: - cpu=8 - image=ubuntu24-full-x64 - disk=large + - spot=false - tag=${{ matrix.config.name }} env: @@ -171,6 +172,7 @@ jobs: - cpu=8 - image=ubuntu24-full-x64 - disk=large + - spot=false - tag=rust-min-deps steps: - uses: runs-on/action@v1 @@ -189,6 +191,7 @@ jobs: - cpu=8 - image=ubuntu24-full-x64 - disk=large + - spot=false - tag=rust-lint steps: - uses: runs-on/action@v1 @@ -257,6 +260,7 @@ jobs: - runs-on=${{ github.run_id }} - family=c7i.8xlarge - image=ubuntu24-full-x64 + - spot=false steps: - uses: runs-on/action@v1 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 6d32272ac0..de98332101 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -14,6 +14,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - tag=io-fuzz + - spot=false steps: - uses: runs-on/action@v1 - uses: actions/checkout@v4 @@ -56,6 +57,7 @@ jobs: - image=ubuntu24-full-x64 - disk=large - tag=ops-fuzz + - spot=false steps: - uses: runs-on/action@v1 - uses: actions/checkout@v4 diff --git a/.github/workflows/generate-tpch-s3.yml b/.github/workflows/generate-tpch-s3.yml index 4d68ecc178..78bf437211 100644 --- a/.github/workflows/generate-tpch-s3.yml +++ b/.github/workflows/generate-tpch-s3.yml @@ -20,6 +20,7 @@ jobs: - runs-on=${{ github.run_id }} - family=m7i.2xlarge - image=ubuntu24-full-x64 + - spot=false env: REMOTE_PATH: s3://vortex-bench-dev-eu/tpch-sf1/ TMPDIR: /work From ff8859c9114539a0cb274f9bebc24022e739ebd5 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Tue, 11 Mar 2025 15:34:24 +0000 Subject: [PATCH 17/17] m7 series --- .github/workflows/ci.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 747f817b05..ff7744b6f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,14 @@ jobs: rust-docs: name: "Rust (docs)" - runs-on: ubuntu-latest + runs-on: + - runs-on=${{ github.run_id }} + - family=m7i+m7i-flex+m7a + - cpu=8 + - image=ubuntu24-full-x64 + - disk=large + - spot=false + - tag=rust-docs steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-rust @@ -114,7 +121,7 @@ jobs: name: "Rust build (${{matrix.config.name}})" runs-on: - runs-on=${{ github.run_id }} - - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - family=m7i+m7i-flex+m7a - cpu=8 - image=ubuntu24-full-x64 - disk=large @@ -168,7 +175,7 @@ jobs: name: "Check build with minimal dependencies" runs-on: - runs-on=${{ github.run_id }} - - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - family=m7i+m7i-flex+m7a - cpu=8 - image=ubuntu24-full-x64 - disk=large @@ -187,7 +194,7 @@ jobs: name: "Rust (lint)" runs-on: - runs-on=${{ github.run_id }} - - family=c7i+c7i-flex+c7a+m7i+m7i-flex+m7a + - family=m7i+m7i-flex+m7a - cpu=8 - image=ubuntu24-full-x64 - disk=large