From f334d1466739896130aa2b6662dfabaf2ddb0669 Mon Sep 17 00:00:00 2001 From: Robert Pack Date: Tue, 10 Dec 2024 15:42:17 +0100 Subject: [PATCH] ci: use official action to setup toolchain --- .github/dependabot.yaml | 14 ++++ .github/workflows/build.yml | 58 +++++---------- .github/workflows/run_integration_test.yml | 12 +--- .github/workflows/semver-checks.yml | 7 +- kernel/src/engine/arrow_data.rs | 83 +++++++++++++--------- 5 files changed, 85 insertions(+), 89 deletions(-) create mode 100644 .github/dependabot.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..529543781 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,14 @@ +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 746a05e37..64b8d33f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,27 +10,20 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install minimal stable with clippy and rustfmt - uses: actions-rs/toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@v1 with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + components: rustfmt - name: format run: cargo fmt -- --check + msrv: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install minimal stable and cargo msrv - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install cargo-msrv shell: bash run: | @@ -41,21 +34,18 @@ jobs: cargo msrv --path derive-macros/ verify --all-features cargo msrv --path ffi/ verify --all-features cargo msrv --path ffi-proc-macros/ verify --all-features + docs: runs-on: ubuntu-latest env: RUSTDOCFLAGS: -D warnings steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install minimal stable - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: build docs run: cargo doc + build: runs-on: ${{ matrix.os }} strategy: @@ -65,18 +55,16 @@ jobs: - ubuntu-latest - windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install minimal stable with clippy and rustfmt - uses: actions-rs/toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@v1 with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + components: clippy - name: build and lint with clippy run: cargo clippy --benches --tests --all-features -- -D warnings - name: lint without default features run: cargo clippy --no-default-features -- -D warnings + test: runs-on: ${{ matrix.os }} strategy: @@ -86,14 +74,9 @@ jobs: - ubuntu-latest - windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install minimal stable with clippy and rustfmt - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: test run: cargo test --workspace --verbose --all-features -- --skip read_table_version_hdfs @@ -109,7 +92,7 @@ jobs: - name: Setup cmake uses: jwlawson/actions-setup-cmake@v2 with: - cmake-version: '3.30.x' + cmake-version: "3.30.x" - name: Install arrow-glib run: | if [ "$RUNNER_OS" == "Linux" ]; then @@ -186,14 +169,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install rust - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - uses: Swatinem/rust-cache@v2 - name: Generate code coverage run: cargo llvm-cov --all-features --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs - name: Upload coverage to Codecov diff --git a/.github/workflows/run_integration_test.yml b/.github/workflows/run_integration_test.yml index 1ff681cf6..73ffd599c 100644 --- a/.github/workflows/run_integration_test.yml +++ b/.github/workflows/run_integration_test.yml @@ -18,17 +18,11 @@ jobs: - name: Skip job for pull requests on Windows if: ${{ matrix.skip }} run: echo "Skipping job for pull requests on Windows." - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ !matrix.skip }} - - name: Install minimal stable rust - if: ${{ !matrix.skip }} - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + - name: Setup rust toolchain if: ${{ !matrix.skip }} + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Run integration tests if: ${{ !matrix.skip }} shell: bash diff --git a/.github/workflows/semver-checks.yml b/.github/workflows/semver-checks.yml index 3f0374452..7bd39b167 100644 --- a/.github/workflows/semver-checks.yml +++ b/.github/workflows/semver-checks.yml @@ -25,12 +25,7 @@ jobs: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Install minimal stable - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install cargo-semver-checks shell: bash run: | diff --git a/kernel/src/engine/arrow_data.rs b/kernel/src/engine/arrow_data.rs index 7c2dd5f40..50a627e5c 100644 --- a/kernel/src/engine/arrow_data.rs +++ b/kernel/src/engine/arrow_data.rs @@ -4,9 +4,11 @@ use crate::{DeltaResult, Error}; use arrow_array::cast::AsArray; use arrow_array::types::{Int32Type, Int64Type}; -use arrow_array::{Array, ArrayRef, GenericListArray, MapArray, OffsetSizeTrait, RecordBatch, StructArray}; -use arrow_schema::{FieldRef, DataType as ArrowDataType}; -use tracing::{debug}; +use arrow_array::{ + Array, ArrayRef, GenericListArray, MapArray, OffsetSizeTrait, RecordBatch, StructArray, +}; +use arrow_schema::{DataType as ArrowDataType, FieldRef}; +use tracing::debug; use std::collections::{HashMap, HashSet}; @@ -138,14 +140,20 @@ impl EngineData for ArrowEngineData { self.data.num_rows() } - fn visit_rows(&self, leaf_columns: &[ColumnName], visitor: &mut dyn RowVisitor) -> DeltaResult<()> { + fn visit_rows( + &self, + leaf_columns: &[ColumnName], + visitor: &mut dyn RowVisitor, + ) -> DeltaResult<()> { // Make sure the caller passed the correct number of column names let leaf_types = visitor.selected_column_names_and_types().1; if leaf_types.len() != leaf_columns.len() { return Err(Error::MissingColumn(format!( "Visitor expected {} column names, but caller passed {}", - leaf_types.len(), leaf_columns.len() - )).with_backtrace()); + leaf_types.len(), + leaf_columns.len() + )) + .with_backtrace()); } // Collect the names of all leaf columns we want to extract, along with their parents, to @@ -154,7 +162,7 @@ impl EngineData for ArrowEngineData { let mut mask = HashSet::new(); for column in leaf_columns { for i in 0..column.len() { - mask.insert(&column[..i+1]); + mask.insert(&column[..i + 1]); } } debug!("Column mask for selected columns {leaf_columns:?} is {mask:#?}"); @@ -162,12 +170,11 @@ impl EngineData for ArrowEngineData { let mut getters = vec![]; Self::extract_columns(&mut vec![], &mut getters, leaf_types, &mask, &self.data)?; if getters.len() != leaf_columns.len() { - return Err(Error::MissingColumn( - format!( - "Visitor expected {} leaf columns, but only {} were found in the data", - leaf_columns.len(), getters.len() - ) - )); + return Err(Error::MissingColumn(format!( + "Visitor expected {} leaf columns, but only {} were found in the data", + leaf_columns.len(), + getters.len() + ))); } visitor.visit(self.len(), &getters) } @@ -185,14 +192,11 @@ impl ArrowEngineData { path.push(field.name().to_string()); if column_mask.contains(&path[..]) { if let Some(struct_array) = column.as_struct_opt() { - debug!("Recurse into a struct array for {}", ColumnName::new(path.iter())); - Self::extract_columns( - path, - getters, - leaf_types, - column_mask, - struct_array, - )?; + debug!( + "Recurse into a struct array for {}", + ColumnName::new(path.iter()) + ); + Self::extract_columns(path, getters, leaf_types, column_mask, struct_array)?; } else if column.data_type() == &ArrowDataType::Null { debug!("Pushing a null array for {}", ColumnName::new(path.iter())); getters.push(&()); @@ -215,16 +219,20 @@ impl ArrowEngineData { col: &'a dyn Array, ) -> DeltaResult<&'a dyn GetData<'a>> { use ArrowDataType::Utf8; - let col_as_list = || if let Some(array) = col.as_list_opt::() { - (array.value_type() == Utf8).then_some(array as _) - } else if let Some(array) = col.as_list_opt::() { - (array.value_type() == Utf8).then_some(array as _) - } else { - None + let col_as_list = || { + if let Some(array) = col.as_list_opt::() { + (array.value_type() == Utf8).then_some(array as _) + } else if let Some(array) = col.as_list_opt::() { + (array.value_type() == Utf8).then_some(array as _) + } else { + None + } + }; + let col_as_map = || { + col.as_map_opt().and_then(|array| { + (array.key_type() == &Utf8 && array.value_type() == &Utf8).then_some(array as _) + }) }; - let col_as_map = || col.as_map_opt().and_then(|array| { - (array.key_type() == &Utf8 && array.value_type() == &Utf8).then_some(array as _) - }); let result: Result<&'a dyn GetData<'a>, _> = match data_type { &DataType::BOOLEAN => { debug!("Pushing boolean array for {}", ColumnName::new(path)); @@ -236,11 +244,15 @@ impl ArrowEngineData { } &DataType::INTEGER => { debug!("Pushing int32 array for {}", ColumnName::new(path)); - col.as_primitive_opt::().map(|a| a as _).ok_or("int") + col.as_primitive_opt::() + .map(|a| a as _) + .ok_or("int") } &DataType::LONG => { debug!("Pushing int64 array for {}", ColumnName::new(path)); - col.as_primitive_opt::().map(|a| a as _).ok_or("long") + col.as_primitive_opt::() + .map(|a| a as _) + .ok_or("long") } DataType::Array(_) => { debug!("Pushing list for {}", ColumnName::new(path)); @@ -252,14 +264,17 @@ impl ArrowEngineData { } data_type => { return Err(Error::UnexpectedColumnType(format!( - "On {}: Unsupported type {data_type}", ColumnName::new(path) + "On {}: Unsupported type {data_type}", + ColumnName::new(path) ))); } }; result.map_err(|type_name| { Error::UnexpectedColumnType(format!( "Type mismatch on {}: expected {}, got {}", - ColumnName::new(path), type_name, col.data_type() + ColumnName::new(path), + type_name, + col.data_type() )) }) }