Merge branch 'develop' into ji/to_duckdb_ext_rs

Author: joseph-isaacs

.github/workflows/bench-pr.yml

@@ -30,7 +30,7 @@ jobs:
       - image=ubuntu24-full-x64
       - disk=large
       - spot=false
-      - tag=${{ matrix.id }}
+      - tag=${{ matrix.benchmark.id }}
     strategy:
       matrix:
         benchmark:
@@ -42,6 +42,8 @@ jobs:
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       # rustup is pre-installed on the ubuntu24-full-x64 image.
 
       # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet

.github/workflows/bench.yml

@@ -15,6 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -36,7 +38,7 @@ jobs:
       - image=ubuntu24-full-x64
       - disk=large
       - spot=false
-      - tag=${{ matrix.id }}
+      - tag=${{ matrix.benchmark.id }}
     strategy:
       matrix:
         benchmark:
@@ -47,6 +49,8 @@ jobs:
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       # rustup is pre-installed on the ubuntu24-full-x64 image.
 
       # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet

.github/workflows/ci.yml

@@ -31,6 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - name: Validate YAML file
         run: |
           # Lint the workflows and yamllint's configuration file.
@@ -44,6 +46,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: spiraldb/actions/.github/actions/[email protected]
       - name: Python Lint - Format
         run: uv run ruff format --check .
@@ -55,6 +59,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: rui314/setup-mold@v1
       - uses: ./.github/actions/setup-rust
@@ -103,16 +109,33 @@ jobs:
 
   rust-docs:
     name: "Rust (docs)"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=8
+      - image=ubuntu24-full-x64
+      - disk=large
+      - spot=false
+      - tag=rust-docs
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/setup-rust
       - name: Docs
         run: cargo doc --no-deps
 
   build-rust:
     name: "Rust build (${{matrix.config.name}})"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=8
+      - image=ubuntu24-full-x64
+      - disk=large
+      - spot=false
+      - tag=${{ matrix.config.name }}
+
     env:
       # disable lints for build, they will be caught in Rust lint job.
       RUSTFLAGS: '-A warnings'
@@ -142,39 +165,59 @@ jobs:
               rustflags: 'RUSTFLAGS="-A warnings --cfg getrandom_backend=\"wasm_js\""'
             args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-tui"
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
+        with:
+          submodules: "recursive"
       - uses: rui314/setup-mold@v1
-      - uses: ./.github/actions/setup-rust
         with:
           targets: ${{matrix.config.target || ''}}
       - uses: ./.github/actions/setup-c++
+      - name: Install wasm32 target
+        if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }}
+        run: rustup target add wasm32-unknown-unknown
       - name: Install cargo-hack
         uses: taiki-e/install-action@cargo-hack
       - name: Rust Build (${{matrix.config.name}})
         run: ${{matrix.config.env.rustflags}} cargo hack ${{matrix.config.command}} --locked ${{matrix.config.args}} --ignore-private
 
   check-min-deps:
     name: "Check build with minimal dependencies"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=8
+      - image=ubuntu24-full-x64
+      - disk=large
+      - spot=false
+      - tag=rust-min-deps
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
+        with:
+          submodules: "recursive"
       - uses: rui314/setup-mold@v1
-      - uses: ./.github/actions/setup-rust
       - uses: ./.github/actions/setup-c++
       - uses: taiki-e/install-action@cargo-hack
       - uses: taiki-e/install-action@cargo-minimal-versions
       - run: cargo minimal-versions check --direct --workspace --ignore-private
 
   rust-lint:
     name: "Rust (lint)"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=8
+      - image=ubuntu24-full-x64
+      - disk=large
+      - spot=false
+      - tag=rust-lint
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
+        with:
+          submodules: "recursive"
       - uses: rui314/setup-mold@v1
-      - uses: ./.github/actions/setup-rust
       - uses: ./.github/actions/setup-c++
       - name: Rust Lint - Format
         run: cargo fmt --all --check
@@ -207,6 +250,8 @@ jobs:
       GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: rui314/setup-mold@v1
       - uses: ./.github/actions/setup-rust
@@ -243,6 +288,8 @@ jobs:
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       # rustup is pre-installed on the ubuntu24-full-x64 image.
 
       - name: Install Codspeed
@@ -282,6 +329,8 @@ jobs:
     continue-on-error: ${{ matrix.checks == 'advisories' }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: EmbarkStudios/cargo-deny-action@v2
         with:
           command: check ${{ matrix.checks }}
@@ -311,6 +360,8 @@ jobs:
     steps:
       - uses: rui314/setup-mold@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/setup-rust
       - uses: taiki-e/install-action@v2
         with:
@@ -327,6 +378,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: rui314/setup-mold@v1
       - uses: ./.github/actions/setup-rust

.github/workflows/docs.yml

@@ -16,6 +16,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: ./.github/actions/setup-rust
       - uses: spiraldb/actions/.github/actions/[email protected]

.github/workflows/fuzz.yml

@@ -14,9 +14,12 @@ jobs:
       - image=ubuntu24-full-x64
       - disk=large
       - tag=io-fuzz
+      - spot=false
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - name: Install cargo fuzz
         run: cargo install cargo-fuzz
       - name: Restore corpus
@@ -56,9 +59,12 @@ jobs:
       - image=ubuntu24-full-x64
       - disk=large
       - tag=ops-fuzz
+      - spot=false
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - name: Install cargo fuzz
         run: cargo install cargo-fuzz
       - name: Restore corpus

.github/workflows/generate-tpch-s3.yml

@@ -1,7 +1,7 @@
 name: "Generate TPC-H files to S3"
 
 on:
-  workflow_dispatch: {}
+  workflow_dispatch: { }
   schedule:
     # 5AM UTC
     - cron: "0 5 * * *"
@@ -20,13 +20,11 @@ jobs:
       - runs-on=${{ github.run_id }}
       - family=m7i.2xlarge
       - image=ubuntu24-full-x64
-    env:
-      REMOTE_PATH: s3://vortex-bench-dev-eu/tpch-sf1/
-      TMPDIR: /work
+      - spot=false
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
-      - uses: ./.github/actions/setup-rust
+        with:
+          submodules: "recursive"
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -37,6 +35,5 @@ jobs:
         run: |
           # We run each query once to make sure we don't upload a file if there's a bug that causes a panic.
           cargo run --release --bin tpch -- --formats parquet,vortex -i1
-          aws s3 cp --recursive bench-vortex/data/tpch/1 $REMOTE_PATH
-          # We can now clear generated data
-          rm -rf bench-vortex/data/
+          aws s3 rm --recursive s3://vortex-bench-dev-eu/tpch-sf1/
+          aws s3 cp --recursive bench-vortex/data/tpch/1 s3://vortex-bench-dev-eu/tpch-sf1/

.github/workflows/release-plz.yml

@@ -23,6 +23,7 @@ jobs:
         with:
           # Needed to pull full commit history for release version number inference
           fetch-depth: 0
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: ./.github/actions/setup-rust
       - name: Run release-plz
@@ -46,6 +47,7 @@ jobs:
         with:
           # Needed to pull full commit history for release version number inference
           fetch-depth: 0
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: ./.github/actions/setup-rust
       - name: Run release-plz
@@ -62,6 +64,8 @@ jobs:
     runs-on: macos-13
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - name: rust-toolchain
         shell: bash
@@ -106,6 +110,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       - uses: astral-sh/setup-uv@v5
       - name: rust-toolchain
         shell: bash

.github/workflows/sql-benchmarks.yml

@@ -31,11 +31,13 @@ jobs:
       - image=ubuntu24-full-x64
       - disk=large
       - spot=false
-      - tag=${{ matrix.id }}
+      - tag=${{ matrix.include.id }}
 
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
       # rustup is pre-installed on the ubuntu24-full-x64 image.
 
       - name: Run ${{ matrix.name }} benchmark

.gitignore

@@ -192,6 +192,7 @@ benchmarks/.out
 *.txt
 *.vortex
 *.parquet
+!docs/_static/example.parquet
 
 # TPC-H benchmarking data
 /data/

.gitmodules

@@ -4,6 +4,9 @@
 [submodule "extension-ci-tools"]
 	path = duckdb-vortex/extension-ci-tools
 	url = https://github.com/duckdb/extension-ci-tools
+[submodule "duckdb-vortex/duckdb-rs"]
+	path = duckdb-vortex/duckdb-rs
+	url = https://github.com/spiraldb/duckdb-rs
 [submodule "duckdb-vortex-rs/extension-ci-tools"]
 	path = duckdb-vortex-rs/extension-ci-tools
 	url = [email protected]:spiraldb/extension-ci-tools.git