Merge remote-tracking branch 'upstream/branch-24.12' into 2412-fix-cl…

…assmethod

.devcontainer/Dockerfile

@@ -31,5 +31,5 @@ ENV PYTHONDONTWRITEBYTECODE="1"
 
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
-ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
+ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
 ENV HISTFILE="/home/coder/.cache/._bash_history"

.pre-commit-config.yaml

@@ -77,7 +77,7 @@ repos:
                 python/cuml/cuml/_thirdparty
         - id: verify-alpha-spec
     - repo: https://github.com/rapidsai/dependency-file-generator
-      rev: v1.13.11
+      rev: v1.16.0
       hooks:
           - id: rapids-dependency-file-generator
             args: ["--clean"]

CHANGELOG.md

@@ -1,3 +1,58 @@
+# cuml 24.10.00 (9 Oct 2024)
+
+## 🚨 Breaking Changes
+
+- Remove old dask-glm based logistic regression ([#6028](https://github.com/rapidsai/cuml/pull/6028)) [@dantegd](https://github.com/dantegd)
+
+## 🐛 Bug Fixes
+
+- Fix train_test_split for string columns ([#6088](https://github.com/rapidsai/cuml/pull/6088)) [@dantegd](https://github.com/dantegd)
+- Stop shadowing free function ([#6076](https://github.com/rapidsai/cuml/pull/6076)) [@vyasr](https://github.com/vyasr)
+- Set default values for conftest options. ([#6067](https://github.com/rapidsai/cuml/pull/6067)) [@bdice](https://github.com/bdice)
+- Add license file to conda packages ([#6061](https://github.com/rapidsai/cuml/pull/6061)) [@raydouglass](https://github.com/raydouglass)
+- Fix np.NAN to np.nan. ([#6056](https://github.com/rapidsai/cuml/pull/6056)) [@bdice](https://github.com/bdice)
+- Reenable `pytest cuml-dask` for CUDA 12.5 wheel CI tests ([#6051](https://github.com/rapidsai/cuml/pull/6051)) [@divyegala](https://github.com/divyegala)
+- Fix for `simplicial_set_embedding` ([#6043](https://github.com/rapidsai/cuml/pull/6043)) [@viclafargue](https://github.com/viclafargue)
+- MAINT: Allow for error message to contain ``np.float32(1.0)`` ([#6030](https://github.com/rapidsai/cuml/pull/6030)) [@seberg](https://github.com/seberg)
+- Stop exporting fill_k kernel as that causes ODR violations ([#6021](https://github.com/rapidsai/cuml/pull/6021)) [@robertmaynard](https://github.com/robertmaynard)
+- Avoid cudf column APIs after cudf.Series disallows column inputs ([#6019](https://github.com/rapidsai/cuml/pull/6019)) [@mroeschke](https://github.com/mroeschke)
+- Use HDBSCAN package pin to `0.8.38` ([#5906](https://github.com/rapidsai/cuml/pull/5906)) [@divyegala](https://github.com/divyegala)
+
+## 📖 Documentation
+
+- Update UMAP doc ([#6064](https://github.com/rapidsai/cuml/pull/6064)) [@viclafargue](https://github.com/viclafargue)
+- Update README in experimental FIL ([#6052](https://github.com/rapidsai/cuml/pull/6052)) [@hcho3](https://github.com/hcho3)
+- add docs for simplicial_set ([#6042](https://github.com/rapidsai/cuml/pull/6042)) [@Intron7](https://github.com/Intron7)
+
+## 🚀 New Features
+
+- TSNE CPU/GPU Interop ([#6063](https://github.com/rapidsai/cuml/pull/6063)) [@divyegala](https://github.com/divyegala)
+- Enable GPU `fit` and CPU `transform` in UMAP ([#6032](https://github.com/rapidsai/cuml/pull/6032)) [@divyegala](https://github.com/divyegala)
+
+## 🛠️ Improvements
+
+- Migrate to use cuVS for vector search ([#6085](https://github.com/rapidsai/cuml/pull/6085)) [@benfred](https://github.com/benfred)
+- Support all-zeroes feature vectors for MG sparse logistic regression ([#6082](https://github.com/rapidsai/cuml/pull/6082)) [@lijinf2](https://github.com/lijinf2)
+- Update update-version.sh to use packaging lib ([#6081](https://github.com/rapidsai/cuml/pull/6081)) [@AyodeAwe](https://github.com/AyodeAwe)
+- Use CI workflow branch 'branch-24.10' again ([#6072](https://github.com/rapidsai/cuml/pull/6072)) [@jameslamb](https://github.com/jameslamb)
+- Update fmt (to 11.0.2) and spdlog (to 1.14.1), add those libraries to libcuml conda host dependencies ([#6071](https://github.com/rapidsai/cuml/pull/6071)) [@jameslamb](https://github.com/jameslamb)
+- Update flake8 to 7.1.1. ([#6070](https://github.com/rapidsai/cuml/pull/6070)) [@bdice](https://github.com/bdice)
+- Add support for Python 3.12, update to umap-learn==0.5.6 ([#6060](https://github.com/rapidsai/cuml/pull/6060)) [@jameslamb](https://github.com/jameslamb)
+- Fix compiler warning about signed vs unsigned ints ([#6053](https://github.com/rapidsai/cuml/pull/6053)) [@hcho3](https://github.com/hcho3)
+- Update rapidsai/pre-commit-hooks ([#6048](https://github.com/rapidsai/cuml/pull/6048)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+- Drop Python 3.9 support ([#6040](https://github.com/rapidsai/cuml/pull/6040)) [@jameslamb](https://github.com/jameslamb)
+- Add use_cuda_wheels matrix entry ([#6038](https://github.com/rapidsai/cuml/pull/6038)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+- Switch debug build to RelWithDebInfo ([#6033](https://github.com/rapidsai/cuml/pull/6033)) [@rongou](https://github.com/rongou)
+- Remove NumPy <2 pin ([#6031](https://github.com/rapidsai/cuml/pull/6031)) [@seberg](https://github.com/seberg)
+- Remove old dask-glm based logistic regression ([#6028](https://github.com/rapidsai/cuml/pull/6028)) [@dantegd](https://github.com/dantegd)
+- [FEA] UMAP API for building with batched NN Descent ([#6022](https://github.com/rapidsai/cuml/pull/6022)) [@jinsolp](https://github.com/jinsolp)
+- Enabling CPU/GPU interop for SVM, DBSCAN and KMeans ([#6020](https://github.com/rapidsai/cuml/pull/6020)) [@viclafargue](https://github.com/viclafargue)
+- Update pre-commit hooks ([#6016](https://github.com/rapidsai/cuml/pull/6016)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+- Improve update-version.sh ([#6014](https://github.com/rapidsai/cuml/pull/6014)) [@bdice](https://github.com/bdice)
+- Use tool.scikit-build.cmake.version, set scikit-build-core minimum-version ([#6012](https://github.com/rapidsai/cuml/pull/6012)) [@jameslamb](https://github.com/jameslamb)
+- Merge branch-24.08 into branch-24.10 ([#5981](https://github.com/rapidsai/cuml/pull/5981)) [@jameslamb](https://github.com/jameslamb)
+- Use CUDA math wheels ([#5966](https://github.com/rapidsai/cuml/pull/5966)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+
 # cuml 24.08.00 (7 Aug 2024)
 
 ## 🐛 Bug Fixes

ci/build_cpp.sh

@@ -15,6 +15,10 @@ rapids-print-env
 
 rapids-logger "Begin cpp build"
 
+sccache --zero-stats
+
 RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcuml
 
+sccache --show-adv-stats
+
 rapids-upload-conda-to-s3 cpp

ci/build_docs.sh

@@ -5,6 +5,9 @@ set -euo pipefail
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+
 rapids-dependency-file-generator \
   --output conda \
   --file-key docs \
@@ -23,11 +26,9 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  cuml libcuml
+  "cuml=${RAPIDS_VERSION}" \
+  "libcuml=${RAPIDS_VERSION}"
 
-export RAPIDS_VERSION="$(rapids-version)"
-export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
-export RAPIDS_VERSION_NUMBER="24.12"
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
 rapids-logger "Build CPP docs"
@@ -44,5 +45,4 @@ mkdir -p "${RAPIDS_DOCS_DIR}/cuml/html"
 mv _html/* "${RAPIDS_DOCS_DIR}/cuml/html"
 popd
 
-rapids-upload-docs
-
+RAPIDS_VERSION_NUMBER="${RAPIDS_VERSION_MAJOR_MINOR}" rapids-upload-docs

ci/build_python.sh

@@ -19,20 +19,28 @@ rapids-logger "Begin py build"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
+sccache --zero-stats
+
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
 RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/cuml
 
+sccache --show-adv-stats
+
 # Build cuml-cpu only in CUDA 11 jobs since it only depends on python
 # version
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
+  sccache --zero-stats
+
   RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --no-test \
   conda/recipes/cuml-cpu
+
+  sccache --show-adv-stats
 fi
 
 rapids-upload-conda-to-s3 python

ci/build_wheel.sh

@@ -21,6 +21,7 @@ cd ${package_dir}
 case "${RAPIDS_CUDA_VERSION}" in
   12.*)
     EXCLUDE_ARGS=(
+      --exclude "libcuvs.so"
       --exclude "libcublas.so.12"
       --exclude "libcublasLt.so.12"
       --exclude "libcufft.so.11"
@@ -32,19 +33,25 @@ case "${RAPIDS_CUDA_VERSION}" in
     EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON"
     ;;
   11.*)
-    EXCLUDE_ARGS=()
+    EXCLUDE_ARGS=(
+      --exclude "libcuvs.so"
+    )
     EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF"
     ;;
 esac
 
-SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/${EXTRA_CMAKE_ARGS}" \
+sccache --zero-stats
+
+SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/;-DUSE_CUVS_WHEEL=ON${EXTRA_CMAKE_ARGS}" \
   python -m pip wheel . \
     -w dist \
-    -vvv \
+    -v \
     --no-deps \
     --disable-pip-version-check
 
+sccache --show-adv-stats
+
 mkdir -p final_dist
 python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/*
 
-RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
+RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist

ci/release/update-version.sh

@@ -40,11 +40,13 @@ echo "${NEXT_FULL_TAG}" > VERSION
 DEPENDENCIES=(
   cudf
   cuml
+  cuvs
   dask-cuda
   dask-cudf
   libcuml
   libcuml-tests
   libcumlprims
+  libcuvs
   libraft-headers
   libraft
   librmm
@@ -72,7 +74,6 @@ for FILE in .github/workflows/*.yaml; do
   # Wheel builds clone cumlprims_mg, update its branch
   sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" "${FILE}"
 done
-sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_SHORT_TAG}\"/g" ci/build_docs.sh
 
 # .devcontainer files
 find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -14,6 +14,7 @@ dependencies:
 - cudatoolkit
 - cudf==24.12.*,>=0.0.0a0
 - cupy>=12.0.0
+- cuvs==24.12.*,>=0.0.0a0
 - cxx-compiler
 - cython>=3.0.0
 - dask-cuda==24.12.*,>=0.0.0a0
@@ -23,7 +24,7 @@ dependencies:
 - fmt>=11.0.2,<12
 - gcc_linux-64=11.*
 - graphviz
-- hdbscan>=0.8.38,<0.8.39
+- hdbscan>=0.8.39,<0.8.40
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython
@@ -39,8 +40,8 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- libcuvs==24.12.*,>=0.0.0a0
 - libraft-headers==24.12.*,>=0.0.0a0
-- libraft==24.12.*,>=0.0.0a0
 - librmm==24.12.*,>=0.0.0a0
 - nbsphinx
 - ninja
@@ -68,7 +69,6 @@ dependencies:
 - scikit-learn==1.5
 - scipy>=1.8.0
 - seaborn
-- setuptools
 - spdlog>=1.14.1,<1.15
 - sphinx-copybutton
 - sphinx-markdown-tables

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -16,6 +16,7 @@ dependencies:
 - cuda-version=12.5
 - cudf==24.12.*,>=0.0.0a0
 - cupy>=12.0.0
+- cuvs==24.12.*,>=0.0.0a0
 - cxx-compiler
 - cython>=3.0.0
 - dask-cuda==24.12.*,>=0.0.0a0
@@ -25,7 +26,7 @@ dependencies:
 - fmt>=11.0.2,<12
 - gcc_linux-64=11.*
 - graphviz
-- hdbscan>=0.8.38,<0.8.39
+- hdbscan>=0.8.39,<0.8.40
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython
@@ -36,8 +37,8 @@ dependencies:
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
+- libcuvs==24.12.*,>=0.0.0a0
 - libraft-headers==24.12.*,>=0.0.0a0
-- libraft==24.12.*,>=0.0.0a0
 - librmm==24.12.*,>=0.0.0a0
 - nbsphinx
 - ninja
@@ -64,7 +65,6 @@ dependencies:
 - scikit-learn==1.5
 - scipy>=1.8.0
 - seaborn
-- setuptools
 - spdlog>=1.14.1,<1.15
 - sphinx-copybutton
 - sphinx-markdown-tables

conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml

@@ -27,8 +27,8 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- libcuvs==24.12.*,>=0.0.0a0
 - libraft-headers==24.12.*,>=0.0.0a0
-- libraft==24.12.*,>=0.0.0a0
 - librmm==24.12.*,>=0.0.0a0
 - ninja
 - nvcc_linux-64=11.8

conda/environments/cpp_all_cuda-118_arch-x86_64.yaml

@@ -25,8 +25,8 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
+- libcuvs==24.12.*,>=0.0.0a0
 - libraft-headers==24.12.*,>=0.0.0a0
-- libraft==24.12.*,>=0.0.0a0
 - librmm==24.12.*,>=0.0.0a0
 - ninja
 - nvcc_linux-64=11.8

conda/environments/cpp_all_cuda-125_arch-x86_64.yaml

@@ -22,8 +22,8 @@ dependencies:
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
+- libcuvs==24.12.*,>=0.0.0a0
 - libraft-headers==24.12.*,>=0.0.0a0
-- libraft==24.12.*,>=0.0.0a0
 - librmm==24.12.*,>=0.0.0a0
 - ninja
 - spdlog>=1.14.1,<1.15

conda/recipes/libcuml/meta.yaml

@@ -70,7 +70,7 @@ requirements:
     {% endif %}
     - fmt {{ fmt_version }}
     - libcumlprims ={{ minor_version }}
-    - libraft ={{ minor_version }}
+    - libcuvs ={{ minor_version }}
     - libraft-headers ={{ minor_version }}
     - librmm ={{ minor_version }}
     - spdlog {{ spdlog_version }}
@@ -116,7 +116,7 @@ outputs:
         - libcusparse
         {% endif %}
         - libcumlprims ={{ minor_version }}
-        - libraft ={{ minor_version }}
+        - libcuvs ={{ minor_version }}
         - librmm ={{ minor_version }}
         - treelite {{ treelite_version }}
     about:

cpp/CMakeLists.txt

@@ -64,8 +64,7 @@ option(SINGLEGPU "Disable all mnmg components and comms libraries" OFF)
 option(USE_CCACHE "Cache build artifacts with ccache" OFF)
 option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
 option(CUDA_STATIC_MATH_LIBRARIES "Statically link the CUDA math libraries" OFF)
-option(CUML_USE_RAFT_STATIC "Build and statically link the RAFT libraries" OFF)
-option(CUML_RAFT_COMPILED "Use libraft shared library" ON)
+option(CUML_USE_CUVS_STATIC "Build and statically link the CUVS library" OFF)
 option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF)
 option(CUML_EXPORT_TREELITE_LINKAGE "Whether to publicly or privately link treelite to libcuml++" OFF)
 option(CUML_USE_CUMLPRIMS_MG_STATIC "Build and statically link the cumlprims_mg library" OFF)
@@ -78,6 +77,7 @@ option(CUML_EXCLUDE_RAFT_FROM_ALL "Exclude RAFT targets from cuML's 'all' target
 option(CUML_EXCLUDE_TREELITE_FROM_ALL "Exclude Treelite targets from cuML's 'all' target" OFF)
 option(CUML_EXCLUDE_CUMLPRIMS_MG_FROM_ALL "Exclude cumlprims_mg targets from cuML's 'all' target" OFF)
 option(CUML_RAFT_CLONE_ON_PIN "Explicitly clone RAFT branch when pinned to non-feature branch" ON)
+option(CUML_CUVS_CLONE_ON_PIN "Explicitly clone CUVS branch when pinned to non-feature branch" ON)
 
 message(VERBOSE "CUML_CPP: Building libcuml_c shared library. Contains the cuML C API: ${BUILD_CUML_C_LIBRARY}")
 message(VERBOSE "CUML_CPP: Building libcuml shared library: ${BUILD_CUML_CPP_LIBRARY}")
@@ -98,7 +98,7 @@ message(VERBOSE "CUML_CPP: Disabling all mnmg components and comms libraries: ${
 message(VERBOSE "CUML_CPP: Cache build artifacts with ccache: ${USE_CCACHE}")
 message(VERBOSE "CUML_CPP: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}")
 message(VERBOSE "CUML_CPP: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}")
-message(VERBOSE "CUML_CPP: Build and statically link RAFT libraries: ${CUML_USE_RAFT_STATIC}")
+message(VERBOSE "CUML_CPP: Build and statically link CUVS libraries: ${CUML_USE_CUVS_STATIC}")
 message(VERBOSE "CUML_CPP: Build and statically link Treelite library: ${CUML_USE_TREELITE_STATIC}")
 
 set(CUML_ALGORITHMS "ALL" CACHE STRING "Experimental: Choose which algorithms are built into libcuml++.so. Can specify individual algorithms or groups in a semicolon-separated list.")
@@ -228,6 +228,9 @@ endif()
 include(cmake/thirdparty/get_cccl.cmake)
 include(cmake/thirdparty/get_rmm.cmake)
 include(cmake/thirdparty/get_raft.cmake)
+if(LINK_CUVS)
+  include(cmake/thirdparty/get_cuvs.cmake)
+endif()
 
 if(LINK_TREELITE)
   include(cmake/thirdparty/get_treelite.cmake)
@@ -442,18 +445,6 @@ if(BUILD_CUML_CPP_LIBRARY)
         src/metrics/kl_divergence.cu
         src/metrics/mutual_info_score.cu
         src/metrics/pairwise_distance.cu
-        src/metrics/pairwise_distance_canberra.cu
-        src/metrics/pairwise_distance_chebyshev.cu
-        src/metrics/pairwise_distance_correlation.cu
-        src/metrics/pairwise_distance_cosine.cu
-        src/metrics/pairwise_distance_euclidean.cu
-        src/metrics/pairwise_distance_hamming.cu
-        src/metrics/pairwise_distance_hellinger.cu
-        src/metrics/pairwise_distance_jensen_shannon.cu
-        src/metrics/pairwise_distance_kl_divergence.cu
-        src/metrics/pairwise_distance_l1.cu
-        src/metrics/pairwise_distance_minkowski.cu
-        src/metrics/pairwise_distance_russell_rao.cu
         src/metrics/r2_score.cu
         src/metrics/rand_index.cu
         src/metrics/silhouette_score.cu
@@ -635,7 +626,7 @@ if(BUILD_CUML_CPP_LIBRARY)
   )
 
   target_link_libraries(${CUML_CPP_TARGET}
-    PUBLIC  rmm::rmm
+    PUBLIC  rmm::rmm ${CUVS_LIB}
             ${_cuml_cpp_public_libs}
     PRIVATE ${_cuml_cpp_private_libs}
   )

cpp/bench/CMakeLists.txt

@@ -50,7 +50,6 @@ if(BUILD_CUML_BENCH)
       benchmark::benchmark
       ${TREELITE_LIBS}
       raft::raft
-      raft::compiled
   )
 
   target_include_directories(${CUML_CPP_BENCH_TARGET}