RAPIDS 24.02 Upgrade

.clang-tidy

@@ -5,7 +5,17 @@
 # Disable modernize-use-trailing-return-type (too many hits)
 # Disable modernize-use-nodiscard (why isnt [[nodiscard]] the default?)
 # Enable identifier in TODO comments
-Checks: -*,readability-identifier-naming,modernize-*,-modernize-use-trailing-return-type,-modernize-use-nodiscard,google-readability-todo
+Checks: >
+  -*,
+  readability-identifier-naming,
+  modernize-*,
+  -modernize-use-trailing-return-type,
+  -modernize-use-nodiscard,
+  google-readability-todo
+
+WarningsAsErrors: >
+  *,
+  -clang-diagnostic-unused-command-line-argument
 
 #WarningsAsErrors: '*'
 HeaderFilterRegex: '.*\/include\/morpheus\/.*'

ci/conda/recipes/morpheus/conda_build_config.yaml

@@ -28,8 +28,5 @@ cuda_compiler_version:
 python:
   - 3.10
 
-boost:
-  - 1.82
-
 rapids_version:
   - 24.02

ci/conda/recipes/morpheus/meta.yaml

@@ -46,81 +46,82 @@ outputs:
     requirements:
       build:
         - {{ compiler("c") }}
-        - {{ compiler("cxx") }}
         - {{ compiler("cuda") }}
-        - automake 1.16.5 # Needed for DOCA build
+        - {{ compiler("cxx") }}
+        - automake =1.16.5 # Needed for DOCA build
         - ccache
-        - cmake 3.25
-        - cuda-cudart-dev {{ cuda_compiler_version }} # Needed by CMake to compile a test application
+        - cmake =3.27
+        - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application
+        - cuda-version {{ cuda_compiler_version }}.*
         - libtool # Needed for DOCA build
-        - ninja
-        - pkg-config # for mrc cmake
+        - ninja =1.11
+        - pkg-config =0.29 # for mrc cmake
+        - sysroot_linux-64 =2.17
       host:
-        - cuda-toolkit {{ cuda_compiler_version }}
+        - cuda-cudart-dev {{ cuda_compiler_version }}.*
+        - cuda-libraries-dev {{ cuda_compiler_version }}.* # required by matx for libcublas-dev
+        - cuda-nvml-dev {{ cuda_compiler_version }}.*
+        - cuda-nvrtc-dev {{ cuda_compiler_version }}.*
+        - cuda-nvtx-dev {{ cuda_compiler_version }}.*
+        - cuda-version {{ cuda_compiler_version }}.*
         - cudf {{ rapids_version }}
-        - cython >=0.29,<0.30
+        - cython =3.0
         - libcudf {{ rapids_version }}
-        - librdkafka 1.9.2
+        - librdkafka >=1.9.2,<1.10.0a0
         - mrc {{ minor_version }}
         - pip
-        - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
-        - pyarrow-hotfix # CVE-2023-47248
-        - pybind11-stubgen 0.10.5
+        - pybind11-stubgen =0.10.5
         - python {{ python }}
-        - rapidjson 1.1
-        - rdma-core >=48 # Needed for DOCA. Compatible with UCX 1.14.1
-        - scikit-build 0.17.1
+        - rapidjson =1.1.0
+        - rdma-core >=48 # Needed for DOCA.
+        - scikit-build =0.17.6
         - versioneer-518
+        - zlib # required to build triton client
       run:
         # Runtime only requirements. This + setup.py is the definitive runtime requirement list
-        - {{ pin_compatible('cuda-cudart', min_pin='x.x', max_pin='x') }}
         - click >=8
-        - configargparse 1.5.*
+        - configargparse =1.5.*
+        - cuda-version {{ cuda_compiler_version }}.*
         - cudf
-        - cudf_kafka {{ rapids_version }}.*
         - cupy # Version determined from cudf
-        - datacompy 0.8.*
+        - datacompy =0.10.*
         - dill
         - distributed
-        - docker-py 5.0.*
+        - docker-py =5.0.*
         - grpcio # Version determined from cudf
         - libmrc
         - libwebp>=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863
         - mlflow>=2.2.1,<3
         - mrc
         - networkx>=2.8
-        - numpydoc 1.4.*
-        - nvtabular {{ rapids_version }}.*
-        - pandas 1.3.*
-        - pluggy 1.0.*
+        - numpydoc =1.5.*
+        - nvtabular =23.08.00
+        - pandas
+        - pluggy =1.3.*
         - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
-        - pyarrow-hotfix # CVE-2023-47248
         - python
-        - python-confluent-kafka 1.9.2
-        - pytorch 2.0.1
+        - python-confluent-kafka >=1.9.2,<1.10.0a0
+        - pytorch
         - pytorch-cuda
-        - requests=2.31
-        - requests-cache=1.1
-        - scikit-learn 1.2.2.*
+        - requests
+        - requests-cache =1.1
+        - scikit-learn =1.3.2.*
         - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3
-        - tqdm 4.*
-        - tritonclient 2.26.*
-        - typing_utils 0.1.*
-        - watchdog 2.1.*
+        - tqdm =4.*
+        - tritonclient =2.34.*
+        - typing_utils =0.1.*
+        - watchdog =3.0.*
         - websockets
-      run_constrained:
-        # Since we dont explicitly require this but other packages might, constrain the versions.
-        - {{ pin_compatible('cudatoolkit', min_pin='x.x', max_pin='x') }}
     test:
       requires:
-        - pynvml
-        - pytest
-        - pytest-cov
-        - pytest-benchmark
         # test that cuml can be installed in the env
         - cuml {{ rapids_version }}.*
+        - pynvml
+        - pytest-asyncio
+        - pytest-benchmark
+        - pytest-cov
+        - pytest =7.4.4
       source_files:
-        - docker/conda/environments/*
         - pyproject.toml
         - scripts/fetch_data.py
         - tests/*

ci/scripts/run_clang_tidy_for_ci.sh

@@ -16,11 +16,7 @@
 
 # set -x
 
-# Call clang-tidy adding warnings-as-errors option. Currently this is not
-# possible with clang-tidy-diff.py until this is merged:
-# https://reviews.llvm.org/D49864
-
 # Also add -fno-caret-diagnostics to prevent clangs own compiler warnings from
 # coming through:
 # https://github.com/llvm/llvm-project/blob/3f3faa36ff3d84af3c3ed84772d7e4278bc44ff1/libc/cmake/modules/LLVMLibCObjectRules.cmake#L226
-${CLANG_TIDY:-clang-tidy} --warnings-as-errors='*' --extra-arg=-fno-caret-diagnostics "$@"
+${CLANG_TIDY:-clang-tidy} --extra-arg=-fno-caret-diagnostics "$@"

cmake/dependencies.cmake

@@ -34,7 +34,7 @@ rapids_find_package(CUDAToolkit REQUIRED
   INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports
 )
 
-rapids_find_package(ZLIB REQUIRED
+rapids_find_package(ZLIB
   BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports
   INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports
 )

conda/environments/all_cuda-121_arch-x86_64.yaml

@@ -13,6 +13,7 @@ dependencies:
 - anyio>=3.7
 - appdirs
 - arxiv=1.4
+- automake
 - benchmark=1.8.3
 - boost-cpp=1.84
 - boto3
@@ -34,10 +35,9 @@ dependencies:
 - cython=3.0
 - dask=2023.12.1
 - datacompy=0.10
-- dill
 - dill=0.3.7
 - distributed=2023.12.1
-- doxygen=1.10.0
+- doxygen=1.9.2
 - elasticsearch==8.9.0
 - exhale=0.3.6
 - feedparser=6.0.10
@@ -55,12 +55,14 @@ dependencies:
 - jsonpatch>=1.33
 - kfp
 - langchain=0.0.190
-- librdkafka=2.2
+- librdkafka>=1.9.2,<1.10.0a0
+- libtool
 - libwebp=1.3.2
 - mlflow=2.9.2
 - mrc=24.03
 - myst-parser=0.18.1
 - nbsphinx
+- networkx=2.8.8
 - newspaper3k=0.2
 - ninja=1.11
 - nlohmann_json=3.9
@@ -75,19 +77,20 @@ dependencies:
 - pluggy=1.3
 - pre-commit
 - protobuf=4.24
-- pybind11-stubgen=0.10
+- pybind11-stubgen=0.10.5
 - pylint=3.0.3
 - pypdf=3.17.4
 - pytest-asyncio
 - pytest-benchmark=4.0
 - pytest-cov
 - pytest=7.4.4
-- python-confluent-kafka=2.2.0
+- python-confluent-kafka>=1.9.2,<1.10.0a0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
 - pytorch=*=*cuda*
 - rapidjson=1.1.0
+- rdma-core>=48
 - requests-cache=1.1
 - s3fs=2023.12.2
 - scikit-build=0.17.6
@@ -106,6 +109,7 @@ dependencies:
 - watchdog=3.0
 - websockets
 - yapf=0.40.1
+- zlib=1.2.13
 - pip:
   - --find-links https://data.dgl.ai/wheels-test/repo.html
   - --find-links https://data.dgl.ai/wheels/cu121/repo.html

conda/environments/dev_cuda-121_arch-x86_64.yaml

@@ -29,8 +29,8 @@ dependencies:
 - cxx-compiler
 - cython=3.0
 - datacompy=0.10
-- dill
-- doxygen=1.10.0
+- dill=0.3.7
+- doxygen=1.9.2
 - elasticsearch==8.9.0
 - exhale=0.3.6
 - feedparser=6.0.10
@@ -43,11 +43,12 @@ dependencies:
 - include-what-you-use=0.20
 - ipython
 - isort
-- librdkafka=2.2
+- librdkafka>=1.9.2,<1.10.0a0
 - mlflow=2.9.2
 - mrc=24.03
 - myst-parser=0.18.1
 - nbsphinx
+- networkx=2.8.8
 - ninja=1.11
 - nlohmann_json=3.9
 - nodejs=18.*
@@ -58,13 +59,13 @@ dependencies:
 - pluggy=1.3
 - pre-commit
 - protobuf=4.24
-- pybind11-stubgen=0.10
+- pybind11-stubgen=0.10.5
 - pylint=3.0.3
 - pytest-asyncio
 - pytest-benchmark=4.0
 - pytest-cov
 - pytest=7.4.4
-- python-confluent-kafka=2.2.0
+- python-confluent-kafka>=1.9.2,<1.10.0a0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -85,6 +86,7 @@ dependencies:
 - watchdog=3.0
 - websockets
 - yapf=0.40.1
+- zlib=1.2.13
 - pip:
   - databricks-connect
   - milvus==2.3.5

conda/environments/examples_cuda-121_arch-x86_64.yaml

@@ -18,7 +18,6 @@ dependencies:
 - cuml=24.02.*
 - dask=2023.12.1
 - datacompy=0.10
-- dill
 - dill=0.3.7
 - distributed=2023.12.1
 - elasticsearch==8.9.0
@@ -31,6 +30,7 @@ dependencies:
 - langchain=0.0.190
 - libwebp=1.3.2
 - mlflow=2.9.2
+- networkx=2.8.8
 - newspaper3k=0.2
 - nodejs=18.*
 - numpydoc=1.5
@@ -40,7 +40,7 @@ dependencies:
 - papermill=2.4.0
 - pip
 - pypdf=3.17.4
-- python-confluent-kafka=2.2.0
+- python-confluent-kafka>=1.9.2,<1.10.0a0
 - python-graphviz
 - python=3.10
 - pytorch-cuda