From 631e0bfbce468081e0ecacf6b5b141f529c6941d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 11 Mar 2025 08:52:09 -0400 Subject: [PATCH 1/6] Better CI feature checks --- .github/workflows/rust.yml | 88 +++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f37e5c9f7ef1..64fe4f763604 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -66,9 +66,12 @@ jobs: # the changes to `Cargo.lock` after building with the updated manifest. cargo check --profile ci --workspace --all-targets --features integration-tests --locked - # cargo check common, functions and substrait with no default features - linux-cargo-check-no-default-features: - name: cargo check no default features + # Check datafusion-common features + # + # Ensure via `cargo check` that the crate can be built with a + # subset of the features packages enabled. + linux-datafusion-common-features: + name: cargo check datafusion-common features needs: linux-build-lib runs-on: ubuntu-latest container: @@ -79,28 +82,63 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Check datafusion without default features - # Some of the test binaries require the parquet feature still - #run: cargo check --all-targets --no-default-features -p datafusion - run: cargo check --profile ci --no-default-features -p datafusion - - - name: Check datafusion-common without default features + - name: Check datafusion-common (no-default-features) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-common + # Note: don't check other feature flags as datafusion-common is not typically used standalone - - name: Check datafusion-functions without default features - run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions - - - name: Check datafusion-substrait without default features + # Check datafusion-substrait features + # + # Ensure via `cargo check` that the crate can be built with a + # subset of the features packages enabled. + linux-datafusion-substrait-features: + name: cargo check datafusion-substrait features + needs: linux-build-lib + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Check datafusion-substrait (no-default-features) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait + - name: Check datafusion-substrait (physical) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical + - name: Check datafusion-substrait (protoc) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc - - name: Check workspace in debug mode - run: cargo check --profile ci --all-targets --workspace + # Check datafusion-proto features + # + # Ensure via `cargo check` that the crate can be built with a + # subset of the features packages enabled. + linux-datafusion-proto-features: + name: cargo check datafusion-proto features + needs: linux-build-lib + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Check datafusion-proto (no-default-features) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto + - name: Check datafusion-proto (json) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=json + - name: Check datafusion-proto (parquet) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=parquet + - name: Check datafusion-proto (avro) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=avro - - name: Check workspace with additional features - run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests - # cargo check datafusion to ensure that the datafusion crate can be built with only a - # subset of the function packages enabled. + # Check datafusion crate features + # + # Ensure via `cargo check` that the crate can be built with a + # subset of the features packages enabled. linux-cargo-check-datafusion: name: cargo check datafusion needs: linux-build-lib @@ -113,6 +151,9 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Check datafusion (no-default-features) + run: cargo check --all-targets --no-default-features -p datafusion + - name: Check datafusion (nested_expressions) run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion @@ -134,8 +175,10 @@ jobs: - name: Check datafusion (string_expressions) run: cargo check --profile ci --no-default-features --features=string_expressions -p datafusion - # cargo check datafusion-functions to ensure that the datafusion-functions crate can be built with - # only a subset of the function packages enabled. + # Check datafusion-functions crate features + # + # Ensure via `cargo check` that the crate can be built with a + # subset of the features packages enabled. linux-cargo-check-datafusion-functions: name: cargo check functions needs: linux-build-lib @@ -148,6 +191,9 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Check datafusion-functions (no-default-features) + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions + - name: Check datafusion-functions (crypto) run: cargo check --profile ci --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions From 5088785144fdfb47bb4c795eab3b4be3e16a8d16 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 11 Mar 2025 09:09:28 -0400 Subject: [PATCH 2/6] adjust feature --- .github/workflows/rust.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 64fe4f763604..4b41a6185f83 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -152,7 +152,9 @@ jobs: with: rust-version: stable - name: Check datafusion (no-default-features) - run: cargo check --all-targets --no-default-features -p datafusion + # Some of the test binaries require the parquet feature still + #run: cargo check --all-targets --no-default-features -p datafusion + run: cargo check --profile ci --no-default-features -p datafusion - name: Check datafusion (nested_expressions) run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion From cafd4285e2d590e7b35f645e75aca698871fb960 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 11 Mar 2025 09:35:21 -0400 Subject: [PATCH 3/6] Install protobuf compiler for checking substrait/protoc --- .github/workflows/rust.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4b41a6185f83..e1bc9f7a6269 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -106,6 +106,8 @@ jobs: run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait - name: Check datafusion-substrait (physical) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical + - name: Install Protobuf Compiler + run: sudo apt-get install -y protobuf-compiler - name: Check datafusion-substrait (protoc) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc From 1c7ffc06a274fadb1885f19634f8b090cdcd2466 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 11 Mar 2025 09:39:18 -0400 Subject: [PATCH 4/6] comment out failing test --- .github/workflows/rust.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e1bc9f7a6269..bb3891dedaf7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -129,8 +129,9 @@ jobs: rust-version: stable - name: Check datafusion-proto (no-default-features) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto - - name: Check datafusion-proto (json) - run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=json + # fails due to https://github.com/apache/datafusion/issues/15157 + #- name: Check datafusion-proto (json) + # run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=json - name: Check datafusion-proto (parquet) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=parquet - name: Check datafusion-proto (avro) From 12c1236e745d53af1a36da790629f197450137f8 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 11 Mar 2025 10:00:19 -0400 Subject: [PATCH 5/6] fix --- .github/workflows/rust.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bb3891dedaf7..7bb78720d05f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -107,7 +107,9 @@ jobs: - name: Check datafusion-substrait (physical) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical - name: Install Protobuf Compiler - run: sudo apt-get install -y protobuf-compiler + run: | + apt-get update + apt-get install -y protobuf-compiler - name: Check datafusion-substrait (protoc) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc From 90839fd6507de651e0f6509e5ee905b57caf5a56 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 11 Mar 2025 10:24:30 -0400 Subject: [PATCH 6/6] fix protoc --- .github/workflows/rust.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7bb78720d05f..525d90d9754f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -106,10 +106,10 @@ jobs: run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait - name: Check datafusion-substrait (physical) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical - - name: Install Protobuf Compiler + - name: Install cmake run: | - apt-get update - apt-get install -y protobuf-compiler + # note the builder setup runs apt-get update / installs protobuf compiler + apt-get install -y cmake - name: Check datafusion-substrait (protoc) run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc