Merge branch 'branch-25.04' into fea/polars/include-file-paths

.github/workflows/build.yaml

@@ -34,6 +34,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+      node_type: "cpu16"
   python-build:
     needs: [cpp-build]
     secrets: inherit
@@ -77,6 +78,7 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
+      node_type: "cpu16"
       script: ci/build_wheel_libcudf.sh
   wheel-publish-libcudf:
     needs: wheel-build-libcudf

.github/workflows/pr.yaml

@@ -24,7 +24,6 @@ jobs:
       - conda-python-cudf-tests
       - conda-python-other-tests
       - conda-java-tests
-      - static-configure
       - conda-notebook-tests
       - docs-build
       - wheel-build-libcudf
@@ -40,6 +39,7 @@ jobs:
       - unit-tests-cudf-pandas
       - pandas-tests
       - pandas-tests-diff
+      - narwhals-tests
       - telemetry-setup
       - third-party-integration-tests-cudf-pandas
     secrets: inherit
@@ -191,16 +191,6 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/test_java.sh"
-  static-configure:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      # Use the wheel container so we can skip conda solves and since our
-      # primary static consumers (Spark) are not in conda anyway.
-      container_image: "rapidsai/ci-wheel:latest"
-      run_script: "ci/configure_cpp_static.sh"
   conda-notebook-tests:
     needs: [conda-python-build, changed-files]
     secrets: inherit
@@ -358,6 +348,20 @@ jobs:
         node_type: "cpu4"
         build_type: pull-request
         run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh"
+  narwhals-tests:
+    needs: [conda-python-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      node_type: "gpu-l4-latest-1"
+      continue-on-error: true
+      container_image: "rapidsai/ci-conda:latest"
+      run_script: ci/test_narwhals.sh
   spark-rapids-jni:
     needs: changed-files
     uses: ./.github/workflows/spark-rapids-jni.yaml

.github/workflows/test.yaml

@@ -46,18 +46,6 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/test_cpp_memcheck.sh"
-  static-configure:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type }}
-      branch: ${{ inputs.branch }}
-      date: ${{ inputs.date }}
-      sha: ${{ inputs.sha }}
-      # Use the wheel container so we can skip conda solves and since our
-      # primary static consumers (Spark) are not in conda anyway.
-      container_image: "rapidsai/ci-wheel:latest"
-      run_script: "ci/configure_cpp_static.sh"
   cpp-linters:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -168,3 +156,14 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: "ci/test_cudf_polars_polars_tests.sh"
+  narwhals-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      node_type: "gpu-l4-latest-1"
+      container_image: "rapidsai/ci-conda:latest"
+      run_script: ci/test_narwhals.sh

.pre-commit-config.yaml

@@ -107,10 +107,6 @@ repos:
           - cmakelang==0.6.13
         verbose: true
         require_serial: true
-        exclude: |
-          (?x)^(
-            cpp/cmake/Modules/FindCUDAToolkit[.]cmake$
-          )
       - id: cmake-lint
         name: cmake-lint
         entry: ./cpp/scripts/run-cmake-format.sh cmake-lint
@@ -122,10 +118,6 @@ repos:
           - cmakelang==0.6.13
         verbose: true
         require_serial: true
-        exclude: |
-          (?x)^(
-            cpp/cmake/Modules/FindCUDAToolkit[.]cmake$
-          )
       - id: doxygen-check
         name: doxygen-check
         entry: ./ci/checks/doxygen.sh
@@ -159,8 +151,7 @@ repos:
           (?x)^(
             cpp/include/cudf_test/cxxopts[.]hpp$|
             cpp/src/io/parquet/ipc/Message_generated[.]h$|
-            cpp/src/io/parquet/ipc/Schema_generated[.]h$|
-            cpp/cmake/Modules/FindCUDAToolkit[.]cmake$
+            cpp/src/io/parquet/ipc/Schema_generated[.]h$
           )
       - id: verify-alpha-spec
       - id: verify-codeowners

ci/build_cpp.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -18,7 +18,7 @@ rapids-logger "Begin cpp build"
 sccache --zero-stats
 
 # With boa installed conda build forward to boa
-RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry build \
     conda/recipes/libcudf
 
 sccache --show-adv-stats

ci/build_python.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -25,15 +25,15 @@ sccache --zero-stats
 # node works correctly
 # With boa installed conda build forwards to the boa builder
 
-RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/pylibcudf
 
 sccache --show-adv-stats
 sccache --zero-stats
 
-RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
@@ -42,27 +42,27 @@ RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
 sccache --show-adv-stats
 sccache --zero-stats
 
-RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/dask-cudf
 
-RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/cudf_kafka
 
 sccache --show-adv-stats
 
-RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/custreamz
 
-RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \

ci/run_cudf_polars_polars_tests.sh

@@ -48,7 +48,9 @@ python -m pytest \
        --cache-clear \
        -m "" \
        -p cudf_polars.testing.plugin \
-       -v \
+       -n 8 \
+       --dist=worksteal \
+       -vv \
        --tb=native \
        $DESELECTED_TESTS_STR \
        "$@" \

ci/run_cudf_polars_pytests.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -13,3 +13,9 @@ python -m pytest --cache-clear "$@" tests
 
 # Test the "dask-experimental" executor
 python -m pytest --cache-clear "$@" tests --executor dask-experimental
+
+# Test the "dask-experimental" executor with Distributed cluster
+# Not all tests pass yet, deselecting by name those that are failing.
+python -m pytest --cache-clear "$@" tests --executor dask-experimental --dask-cluster \
+    -k "not test_groupby_maintain_order_random and not test_scan_csv_multi and not test_select_literal_series" \
+    --cov-fail-under=89  # Override coverage, Distributed cluster coverage not yet 100%

ci/test_cudf_polars_polars_tests.sh

@@ -26,6 +26,8 @@ git clone https://github.com/pola-rs/polars.git --branch "${TAG}" --depth 1
 
 # Install requirements for running polars tests
 rapids-logger "Install polars test requirements"
+# TODO: Remove sed command when polars-cloud supports 1.23
+sed -i '/^polars-cloud$/d' polars/py-polars/requirements-dev.txt
 rapids-pip-retry install -r polars/py-polars/requirements-dev.txt -r polars/py-polars/requirements-ci.txt
 
 # shellcheck disable=SC2317

ci/test_narwhals.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright (c) 2025, NVIDIA CORPORATION.
+
+# Support invoking test_python_cudf.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ || exit 1
+
+# Common setup steps shared by Python test jobs
+source ./ci/test_python_common.sh test_python_narwhals
+
+rapids-logger "Check GPU usage"
+nvidia-smi
+rapids-print-env
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+rapids-logger "pytest narwhals"
+git clone https://github.com/narwhals-dev/narwhals --depth=1
+pushd narwhals || exit 1
+rapids-pip-retry install -U -e ".[dev]"
+
+rapids-logger "Check narwhals versions"
+python -c "import narwhals; print(narwhals.show_versions())"
+
+rapids-logger "Run narwhals tests for cuDF"
+python -m pytest \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-narwhals.xml" \
+    -p cudf.testing.narwhals_test_plugin \
+    --numprocesses=8 \
+    --dist=worksteal \
+    --constructors=cudf
+
+rapids-logger "Run narwhals tests for cuDF Polars"
+NARWHALS_POLARS_GPU=1 python -m pytest \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-narwhals.xml" \
+    --numprocesses=8 \
+    --dist=worksteal \
+    --constructors=polars[lazy]
+
+popd || exit 1
+
+rapids-logger "Test script exiting with value: $EXITCODE"
+exit ${EXITCODE}

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -15,7 +15,7 @@ dependencies:
 - cachetools
 - clang-tools=16.0.6
 - clang==16.0.6
-- cmake>=3.26.4,!=3.30.0
+- cmake>=3.30.4
 - cramjam
 - cubinlinker
 - cuda-nvtx=11.8
@@ -54,19 +54,19 @@ dependencies:
 - nbsphinx
 - ninja
 - notebook
-- numba-cuda>=0.2.0,<0.3.0a0
-- numba>=0.59.1,<0.61.0a0
-- numpy>=1.23,<3.0a0
+- numba-cuda>=0.4.0,<0.5.0a0
+- numba>=0.59.1,<0.62.0a0
+- numpy>=1.23,<2.1
 - numpydoc
 - nvcc_linux-64=11.8
-- nvcomp==4.1.0.6
+- nvcomp==4.2.0.11
 - nvtx>=0.2.1
 - openpyxl
 - packaging
 - pandas
 - pandas>=2.0,<2.2.4dev0
 - pandoc
-- polars>=1.20,<1.23
+- polars>=1.20,<1.24
 - pre-commit
 - ptxcompiler
 - pyarrow>=14.0.0,<20.0.0a0

conda/environments/all_cuda-128_arch-x86_64.yaml

@@ -15,7 +15,7 @@ dependencies:
 - cachetools
 - clang-tools=16.0.6
 - clang==16.0.6
-- cmake>=3.26.4,!=3.30.0
+- cmake>=3.30.4
 - cramjam
 - cuda-cudart-dev
 - cuda-nvcc
@@ -53,18 +53,18 @@ dependencies:
 - nbsphinx
 - ninja
 - notebook
-- numba-cuda>=0.2.0,<0.3.0a0
-- numba>=0.59.1,<0.61.0a0
-- numpy>=1.23,<3.0a0
+- numba-cuda>=0.4.0,<0.5.0a0
+- numba>=0.59.1,<0.62.0a0
+- numpy>=1.23,<2.1
 - numpydoc
-- nvcomp==4.1.0.6
+- nvcomp==4.2.0.11
 - nvtx>=0.2.1
 - openpyxl
 - packaging
 - pandas
 - pandas>=2.0,<2.2.4dev0
 - pandoc
-- polars>=1.20,<1.23
+- polars>=1.20,<1.24
 - pre-commit
 - pyarrow>=14.0.0,<20.0.0a0
 - pydata-sphinx-theme>=0.15.4

conda/recipes/cudf-polars/meta.yaml

@@ -43,7 +43,7 @@ requirements:
   run:
     - python
     - pylibcudf ={{ version }}
-    - polars >=1.20,<1.23
+    - polars >=1.20,<1.24
     - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
 
 test:

conda/recipes/cudf/conda_build_config.yaml

@@ -13,7 +13,7 @@ c_stdlib_version:
   - "2.28"
 
 cmake_version:
-  - ">=3.26.4,!=3.30.0"
+  - ">=3.30.4"
 
 cuda_compiler:
   - cuda-nvcc  # [not os.environ.get("RAPIDS_CUDA_VERSION", "").startswith("11")]