Skip to content

Commit

Permalink
Merge branch 'branch-25.04' into fix/disable_thread_safety
Browse files Browse the repository at this point in the history
  • Loading branch information
Matt711 authored Feb 13, 2025
2 parents 8d15be7 + 9ead47b commit 7bebbb3
Show file tree
Hide file tree
Showing 160 changed files with 2,468 additions and 2,150 deletions.
16 changes: 16 additions & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ jobs:
- pandas-tests
- pandas-tests-diff
- telemetry-setup
- third-party-integration-tests-cudf-pandas
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
if: always()
Expand Down Expand Up @@ -141,6 +142,7 @@ jobs:
with:
build_type: pull-request
run_script: "ci/cpp_linters.sh"
node_type: "cpu16"
conda-cpp-checks:
needs: conda-cpp-build
secrets: inherit
Expand Down Expand Up @@ -321,6 +323,20 @@ jobs:
matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
build_type: pull-request
script: ci/cudf_pandas_scripts/run_tests.sh
third-party-integration-tests-cudf-pandas:
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
build_type: pull-request
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
node_type: "gpu-l4-latest-1"
continue-on-error: true
container_image: "rapidsai/ci-conda:latest"
run_script: |
ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml
pandas-tests:
# run the Pandas unit tests using PR branch
needs: [wheel-build-cudf, changed-files]
Expand Down
648 changes: 648 additions & 0 deletions CHANGELOG.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions ci/build_wheel_cudf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ python -m auditwheel repair \
--exclude libcudf.so \
--exclude libnvcomp.so \
--exclude libkvikio.so \
--exclude librapids_logger.so \
-w ${package_dir}/final_dist \
${package_dir}/dist/*

Expand Down
1 change: 1 addition & 0 deletions ci/build_wheel_libcudf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ mkdir -p ${package_dir}/final_dist
python -m auditwheel repair \
--exclude libnvcomp.so.4 \
--exclude libkvikio.so \
--exclude librapids_logger.so \
-w ${package_dir}/final_dist \
${package_dir}/dist/*

Expand Down
1 change: 1 addition & 0 deletions ci/build_wheel_pylibcudf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ python -m auditwheel repair \
--exclude libcudf.so \
--exclude libnvcomp.so \
--exclude libkvikio.so \
--exclude librapids_logger.so \
-w ${package_dir}/final_dist \
${package_dir}/dist/*

Expand Down
32 changes: 24 additions & 8 deletions ci/cudf_pandas_scripts/third-party-integration/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,38 @@ main() {
LIBS=${LIBS#[}
LIBS=${LIBS%]}

if [ "$RAPIDS_BUILD_TYPE" == "pull-request" ]; then
rapids-logger "Downloading artifacts from this pr jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
fi

ANY_FAILURES=0

for lib in ${LIBS//,/ }; do
lib=$(echo "$lib" | tr -d '""')
echo "Running tests for library $lib"

CUDA_VERSION=$(if [ "$lib" = "tensorflow" ]; then echo "11.8"; else echo "${RAPIDS_CUDA_VERSION%.*}"; fi)

. /opt/conda/etc/profile.d/conda.sh

rapids-logger "Generate Python testing dependencies"
rapids-dependency-file-generator \
--config "$dependencies_yaml" \
--output conda \
--file-key "test_${lib}" \
--matrix "cuda=${CUDA_VERSION};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
# Check the value of RAPIDS_BUILD_TYPE
if [ "$RAPIDS_BUILD_TYPE" == "pull-request" ]; then
rapids-logger "Generate Python testing dependencies"
rapids-dependency-file-generator \
--config "$dependencies_yaml" \
--output conda \
--file-key "test_${lib}" \
--matrix "cuda=${CUDA_VERSION};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" \
--prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
else
rapids-logger "Generate Python testing dependencies"
rapids-dependency-file-generator \
--config "$dependencies_yaml" \
--output conda \
--file-key "test_${lib}" \
--matrix "cuda=${CUDA_VERSION};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
fi

rapids-mamba-retry env create --yes -f env.yaml -n test

Expand Down
1 change: 1 addition & 0 deletions ci/release/update-version.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_dask_cudf.sh
DEPENDENCIES=(
cudf
cudf_kafka
cudf-polars
cugraph
cuml
custreamz
Expand Down
8 changes: 4 additions & 4 deletions ci/test_python_common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ set -euo pipefail

RAPIDS_VERSION="$(rapids-version)"

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-logger "Generate Python testing dependencies"

ENV_YAML_DIR="$(mktemp -d)"
Expand All @@ -26,10 +30,6 @@ set +u
conda activate test
set -u

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}
RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${RESULTS_DIR}/coverage-results"}
Expand Down
12 changes: 1 addition & 11 deletions ci/test_python_other.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,9 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
# Common setup steps shared by Python test jobs
source ./ci/test_python_common.sh test_python_other

RAPIDS_VERSION="$(rapids-version)"

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
"dask-cudf=${RAPIDS_VERSION}" \
"cudf_kafka=${RAPIDS_VERSION}" \
"custreamz=${RAPIDS_VERSION}" \
"cudf-polars=${RAPIDS_VERSION}"

rapids-logger "Check GPU usage"
nvidia-smi

rapids-print-env
EXITCODE=0
trap "EXITCODE=1" ERR
set +e
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-x86_64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ dependencies:
- doxygen=1.9.1
- fastavro>=0.22.9
- flatbuffers==24.3.25
- fmt>=11.0.2,<12
- fsspec>=0.6.0
- gcc_linux-64=11.*
- hypothesis
Expand Down Expand Up @@ -83,6 +82,7 @@ dependencies:
- python>=3.10,<3.13
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==25.4.*,>=0.0.0a0
- rapids-logger==0.1.*,>=0.0.0a0
- rich
- rmm==25.4.*,>=0.0.0a0
- s3fs>=2022.3.0
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-128_arch-x86_64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ dependencies:
- doxygen=1.9.1
- fastavro>=0.22.9
- flatbuffers==24.3.25
- fmt>=11.0.2,<12
- fsspec>=0.6.0
- gcc_linux-64=13.*
- hypothesis
Expand Down Expand Up @@ -82,6 +81,7 @@ dependencies:
- pytorch>=2.4.0
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==25.4.*,>=0.0.0a0
- rapids-logger==0.1.*,>=0.0.0a0
- rich
- rmm==25.4.*,>=0.0.0a0
- s3fs>=2022.3.0
Expand Down
3 changes: 0 additions & 3 deletions conda/recipes/libcudf/conda_build_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,6 @@ dlpack_version:
librdkafka_version:
- ">=2.5.0,<2.6.0a0"

fmt_version:
- ">=11.0.2,<12"

flatbuffers_version:
- "=24.3.25"

Expand Down
3 changes: 2 additions & 1 deletion conda/recipes/libcudf/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ requirements:
- nvcomp {{ nvcomp_version }}
- dlpack {{ dlpack_version }}
- librdkafka {{ librdkafka_version }}
- fmt {{ fmt_version }}
- flatbuffers {{ flatbuffers_version }}
- rapids-logger =0.1
- zlib {{ zlib_version }}

outputs:
Expand Down Expand Up @@ -99,6 +99,7 @@ outputs:
- librmm ={{ minor_version }}
- libkvikio ={{ minor_version }}
- dlpack {{ dlpack_version }}
- rapids-logger =0.1
test:
commands:
- test -f $PREFIX/lib/libcudf.so
Expand Down
34 changes: 7 additions & 27 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@ option(CUDA_ENABLE_LINEINFO
option(CUDA_WARNINGS_AS_ERRORS "Enable -Werror=all-warnings for all CUDA compilation" ON)
# cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking
option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
option(CUDA_STATIC_CUFILE "Statically link cuFile" OFF)

set(DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL ON)
if(CUDA_STATIC_RUNTIME OR NOT BUILD_SHARED_LIBS)
Expand Down Expand Up @@ -275,8 +274,8 @@ endif()
rapids_cpm_init()

include(${rapids-cmake-dir}/cpm/rapids_logger.cmake)
rapids_cpm_rapids_logger()
rapids_make_logger(cudf EXPORT_SET cudf-exports LOGGER_DEFAULT_LEVEL WARN)
rapids_cpm_rapids_logger(BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports)
create_logger_macros(CUDF "cudf::default_logger()" include/cudf)

# find jitify
include(cmake/thirdparty/get_jitify.cmake)
Expand All @@ -302,8 +301,6 @@ endif()
include(cmake/Modules/JitifyPreprocessKernels.cmake)
# find KvikIO
include(cmake/thirdparty/get_kvikio.cmake)
# find fmt
include(cmake/thirdparty/get_fmt.cmake)
# find nanoarrow
include(cmake/thirdparty/get_nanoarrow.cmake)
# find thread_pool
Expand Down Expand Up @@ -466,7 +463,6 @@ add_library(
src/hash/xxhash_64.cu
src/interop/dlpack.cpp
src/interop/arrow_utilities.cpp
src/interop/decimal_conversion_utilities.cu
src/interop/to_arrow_device.cu
src/interop/to_arrow_host.cu
src/interop/from_arrow_device.cu
Expand Down Expand Up @@ -549,7 +545,6 @@ add_library(
src/io/utilities/data_casting.cu
src/io/utilities/data_sink.cpp
src/io/utilities/datasource.cpp
src/io/utilities/file_io_utilities.cpp
src/io/utilities/row_selection.cpp
src/io/utilities/type_inference.cu
src/io/utilities/trie.cu
Expand Down Expand Up @@ -779,6 +774,7 @@ add_library(
src/utilities/default_stream.cpp
src/utilities/host_memory.cpp
src/utilities/linked_column.cpp
src/utilities/logger.cpp
src/utilities/prefetch.cpp
src/utilities/stacktrace.cpp
src/utilities/stream_pool.cpp
Expand Down Expand Up @@ -924,15 +920,6 @@ target_compile_definitions(
# Enable remote IO through KvikIO
target_compile_definitions(cudf PRIVATE $<$<BOOL:${CUDF_KVIKIO_REMOTE_IO}>:CUDF_KVIKIO_REMOTE_IO>)

# Enable cuFile support
set(_cufile_suffix)
if(CUDA_STATIC_CUFILE)
set(_cufile_suffix _static)
endif()
if(TARGET CUDA::cuFile${_cufile_suffix})
target_compile_definitions(cudf PRIVATE CUDF_CUFILE_FOUND)
endif()

# Remove this after upgrading to a CCCL that has a proper CMake option. See
# https://github.com/NVIDIA/cccl/pull/2844
target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1)
Expand All @@ -943,16 +930,9 @@ add_dependencies(cudf jitify_preprocess_run)
# Specify the target module library dependencies
target_link_libraries(
cudf
PUBLIC CCCL::CCCL rmm::rmm rmm::rmm_logger $<BUILD_LOCAL_INTERFACE:BS::thread_pool> cudf_logger
PRIVATE $<BUILD_LOCAL_INTERFACE:nvtx3::nvtx3-cpp>
cuco::cuco
ZLIB::ZLIB
nvcomp::nvcomp
kvikio::kvikio
$<TARGET_NAME_IF_EXISTS:CUDA::cuFile${_cufile_suffix}>
nanoarrow
rmm::rmm_logger_impl
cudf_logger_impl
PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $<BUILD_LOCAL_INTERFACE:BS::thread_pool>
PRIVATE $<BUILD_LOCAL_INTERFACE:nvtx3::nvtx3-cpp> cuco::cuco ZLIB::ZLIB nvcomp::nvcomp
kvikio::kvikio nanoarrow
)

# Add Conda library, and include paths if specified
Expand Down Expand Up @@ -1108,7 +1088,7 @@ if(CUDF_BUILD_STREAMS_TEST_UTIL)
${_tgt} PRIVATE "$<BUILD_INTERFACE:$<$<COMPILE_LANGUAGE:CXX>:${CUDF_CXX_FLAGS}>>"
)
target_include_directories(${_tgt} PRIVATE "$<BUILD_INTERFACE:${CUDF_SOURCE_DIR}/include>")
target_link_libraries(${_tgt} PUBLIC CUDA::cudart rmm::rmm rmm::rmm_logger rmm::rmm_logger_impl)
target_link_libraries(${_tgt} PUBLIC CUDA::cudart rmm::rmm)
if(CUDF_BUILD_STACKTRACE_DEBUG)
target_link_libraries(${_tgt} PRIVATE cudf_backtrace)
endif()
Expand Down
11 changes: 5 additions & 6 deletions cpp/benchmarks/common/generate_input.cu
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2020-2024, NVIDIA CORPORATION.
* Copyright (c) 2020-2025, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -46,7 +46,6 @@
#include <thrust/execution_policy.h>
#include <thrust/fill.h>
#include <thrust/for_each.h>
#include <thrust/functional.h>
#include <thrust/gather.h>
#include <thrust/iterator/counting_iterator.h>
#include <thrust/iterator/transform_iterator.h>
Expand Down Expand Up @@ -511,7 +510,7 @@ std::unique_ptr<cudf::column> create_random_column(data_profile const& profile,
auto [result_bitmask, null_count] =
cudf::detail::valid_if(null_mask.begin(),
null_mask.end(),
thrust::identity<bool>{},
cuda::std::identity{},
cudf::get_default_stream(),
cudf::get_current_device_resource_ref());

Expand Down Expand Up @@ -600,7 +599,7 @@ std::unique_ptr<cudf::column> create_random_utf8_string_column(data_profile cons
auto [result_bitmask, null_count] =
profile.get_null_probability().has_value()
? cudf::detail::valid_if(
null_mask.begin(), null_mask.end() - 1, thrust::identity<bool>{}, stream, mr)
null_mask.begin(), null_mask.end() - 1, cuda::std::identity{}, stream, mr)
: std::pair{rmm::device_buffer{}, 0};

return cudf::make_strings_column(
Expand Down Expand Up @@ -693,7 +692,7 @@ std::unique_ptr<cudf::column> create_random_column<cudf::struct_view>(data_profi
auto valids = valid_dist(engine, num_rows);
return cudf::detail::valid_if(valids.begin(),
valids.end(),
thrust::identity<bool>{},
cuda::std::identity{},
cudf::get_default_stream(),
cudf::get_current_device_resource_ref());
}
Expand Down Expand Up @@ -787,7 +786,7 @@ std::unique_ptr<cudf::column> create_random_column<cudf::list_view>(data_profile

auto [null_mask, null_count] = cudf::detail::valid_if(valids.begin(),
valids.end(),
thrust::identity<bool>{},
cuda::std::identity{},
cudf::get_default_stream(),
cudf::get_current_device_resource_ref());
list_column = cudf::make_lists_column(
Expand Down
3 changes: 2 additions & 1 deletion cpp/benchmarks/hashing/partition.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2023, NVIDIA CORPORATION.
* Copyright (c) 2019-2025, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -21,6 +21,7 @@
#include <cudf/partitioning.hpp>

#include <algorithm>
#include <numeric>

class Hashing : public cudf::benchmark {};

Expand Down
3 changes: 2 additions & 1 deletion cpp/benchmarks/io/text/multibyte_split.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2021-2024, NVIDIA CORPORATION.
* Copyright (c) 2021-2025, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -38,6 +38,7 @@
#include <cstdio>
#include <fstream>
#include <memory>
#include <numeric>
#include <random>

temp_directory const temp_dir("cudf_nvbench");
Expand Down
Loading

0 comments on commit 7bebbb3

Please sign in to comment.