Fix or silence warnings emitted during tests

ci/conda/recipes/morpheus/meta.yaml

@@ -92,6 +92,7 @@ outputs:
         - pandas 1.3.*
         - pluggy 1.0.*
         - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+        - pyarrow-hotfix # CVE-2023-47248
         - python
         - python-confluent-kafka 1.9.2
         - pytorch 2.0.1

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -1,14 +1,13 @@
 # This file is generated by `rapids-dependency-file-generator`.
-# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+# To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 channels:
-- conda-forge
-- huggingface
 - rapidsai
 - nvidia/label/cuda-11.8.0
 - nvidia
-- rapidsai-nightly
 - nvidia/label/dev
+- huggingface
 - pytorch
+- conda-forge
 - dglteam/label/cu118
 dependencies:
 - appdirs
@@ -17,6 +16,8 @@ dependencies:
 - boost-cpp=1.82
 - boto3
 - breathe=4.34.0
+- cachetools=5.0.0
+- ccache>=3.7
 - clangdev=16
 - click >=8
 - click>=8
@@ -34,7 +35,7 @@ dependencies:
 - cython=0.29.24
 - dask>=2023.1.1
 - datacompy=0.8
-- dgl=1.0.2
+- dgl=1.1.1
 - dill
 - dill=0.3.6
 - distributed>=2023.1.1
@@ -52,6 +53,7 @@ dependencies:
 - include-what-you-use=0.20
 - ipython
 - isort
+- kafka-python=2.0
 - langchain=0.0.190
 - librdkafka=1.9.2
 - libwebp>=1.3.2
@@ -65,14 +67,20 @@ dependencies:
 - ninja=1.10
 - nlohmann_json=3.9
 - nodejs=18.*
+- numpy>=1.21,<1.25
 - numpydoc=1.4
 - nvtabular=23.06
 - openai=0.28
+- pandas=1.3
 - papermill=2.3.4
 - pip
 - pluggy=1.0
+- port-for=0.7
 - pre-commit
 - protobuf=4.21.*
+- py4j=0.10
+- pyarrow * *_cuda
+- pyarrow-hotfix
 - pybind11-stubgen=0.10
 - pylint>=2.17.4,<2.18
 - pypdf=3.16
@@ -94,6 +102,7 @@ dependencies:
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
+- sysroot_linux-64=2.17
 - tqdm=4
 - transformers=4.30.2
 - tritonclient=2.26
@@ -109,7 +118,8 @@ dependencies:
   - grpcio-status==1.58
   - milvus==2.3.2
   - nemollm
-  - pyarrow_hotfix
   - pymilvus==2.3.2
   - pytest-kafka==0.6.0
-name: all_cuda-118_arch-x86_64
+  - rapids-dependency-file-generator
+name: all_cuda-118_arch-x86_64_py-310
+

conda/environments/dev_cuda-118_arch-x86_64.yaml

@@ -1,20 +1,21 @@
 # This file is generated by `rapids-dependency-file-generator`.
-# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+# To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 channels:
-- conda-forge
-- huggingface
 - rapidsai
 - nvidia/label/cuda-11.8.0
 - nvidia
-- rapidsai-nightly
 - nvidia/label/dev
+- huggingface
 - pytorch
+- conda-forge
 - dglteam/label/cu118
 dependencies:
 - appdirs
 - benchmark=1.6.0
 - boost-cpp=1.82
 - breathe=4.34.0
+- cachetools=5.0.0
+- ccache>=3.7
 - clangdev=16
 - click >=8
 - click>=8
@@ -43,6 +44,7 @@ dependencies:
 - include-what-you-use=0.20
 - ipython
 - isort
+- kafka-python=2.0
 - librdkafka=1.9.2
 - mlflow>=2.2.1,<3
 - mrc=24.03
@@ -52,12 +54,18 @@ dependencies:
 - ninja=1.10
 - nlohmann_json=3.9
 - nodejs=18.*
+- numpy>=1.21,<1.25
 - numpydoc=1.4
 - nvtabular=23.06
+- pandas=1.3
 - pip
 - pluggy=1.0
+- port-for=0.7
 - pre-commit
 - protobuf=4.21.*
+- py4j=0.10
+- pyarrow * *_cuda
+- pyarrow-hotfix
 - pybind11-stubgen=0.10
 - pylint>=2.17.4,<2.18
 - pytest
@@ -75,6 +83,7 @@ dependencies:
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
+- sysroot_linux-64=2.17
 - tqdm=4
 - tritonclient=2.26
 - typing_utils=0.1
@@ -85,7 +94,8 @@ dependencies:
 - pip:
   - databricks-connect
   - milvus==2.3.2
-  - pyarrow_hotfix
   - pymilvus==2.3.2
   - pytest-kafka==0.6.0
-name: dev_cuda-118_arch-x86_64
+  - rapids-dependency-file-generator
+name: dev_cuda-118_arch-x86_64_py-310
+

dependencies.yaml

@@ -107,14 +107,13 @@ files:
 
 
 channels:
-  - conda-forge
-  - huggingface
   - rapidsai
   - nvidia/label/cuda-11.8.0
   - nvidia
-  - rapidsai-nightly
   - nvidia/label/dev
+  - huggingface
   - pytorch
+  - conda-forge
   - dglteam/label/cu118
 
 dependencies:
@@ -126,6 +125,8 @@ dependencies:
           - mrc=24.03 # should this be in build, or somewhere else?
           - gcc_linux-64=11.2
           - gxx_linux-64=11.2
+          - cachetools=5.0.0
+          - ccache>=3.7
           - cmake=3.25
           - boost-cpp=1.82
           - cuda-nvcc
@@ -135,10 +136,14 @@ dependencies:
           - librdkafka=1.9.2
           - ninja=1.10
           - nlohmann_json=3.9
+          - pandas=1.3
           - protobuf=4.21.*
+          - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+          - pyarrow-hotfix
           - pybind11-stubgen=0.10
           - rapidjson=1.1.0
           - scikit-build=0.17.1
+          - sysroot_linux-64=2.17
           - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21
           - ucx=1.14
 
@@ -164,6 +169,9 @@ dependencies:
           - isort
           - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support
           - yapf=0.40.1
+          - pip
+          - pip:
+            - rapids-dependency-file-generator
 
   docs:
     common:
@@ -196,7 +204,7 @@ dependencies:
           - boto3
           - cuml=23.06
           - dask>=2023.1.1
-          - dgl=1.0.2
+          - dgl=1.1.1
           - dill=0.3.6
           - distributed>=2023.1.1
           - huggingface_hub=0.10.1 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762
@@ -238,8 +246,12 @@ dependencies:
           - grpcio
           - mlflow>=2.2.1,<3
           - nb_conda_kernels
+          # Avoids numerous warnings triggered by cudf, recent versions of cudf impost this version requirement
+          - numpy>=1.21,<1.25
           - numpydoc=1.4
           - nvtabular=23.06
+          - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+          - pyarrow-hotfix
           - python-confluent-kafka=1.9.2
           - python-graphviz
           - pytorch-cuda
@@ -251,11 +263,17 @@ dependencies:
           - typing_utils=0.1
           - watchdog=2.1
           - websockets
+
+          ####### Pip Transitive Dependencies (keep sorted!) #######
+          # These are dependencies that are available on conda, but are required by the pip packages listed below. Its much
+          # better to install them with conda than pip to allow for better dependency resolution.
+          - kafka-python=2.0
+          - port-for=0.7
+          - py4j=0.10
           - pip
           - pip:
             - databricks-connect
             - milvus==2.3.2
-            - pyarrow_hotfix
             - pymilvus==2.3.2
 
   test_python_morpheus:

docker/conda/environments/cuda11.8_dev.yml

@@ -74,6 +74,8 @@ dependencies:
     - ninja=1.10
     - nodejs=18.*
     - numba>=0.56.2
+    # Avoids numerous warnings triggered by cudf, recent versions of cudf impost this version
+    - numpy>=1.21,<1.25
     - numpydoc=1.4
     - nvtabular=23.06
     - pandas=1.3
@@ -82,6 +84,7 @@ dependencies:
     - pluggy=1.0
     - protobuf=4.21.*
     - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+    - pyarrow-hotfix # CVE-2023-47248. See morpheus/__init__.py for more details
     - pybind11-stubgen=0.10.5
     - pydot
     - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support
@@ -123,6 +126,5 @@ dependencies:
         # Add additional dev dependencies here
         - databricks-connect
         - milvus==2.3.2
-        - pyarrow_hotfix # CVE-2023-47248. See morpheus/__init__.py for more details
         - pymilvus==2.3.2
         - pytest-kafka==0.6.0

docker/conda/environments/cuda11.8_examples.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,7 +32,7 @@ dependencies:
     - boto3
     - cuml=23.06
     - dask>=2023.1.1
-    - dgl=1.0.2
+    - dgl=1.1.1
     - dill=0.3.6
     - distributed>=2023.1.1
     - huggingface_hub=0.10.1 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762

examples/gnn_fraud_detection_pipeline/requirements.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,4 +21,4 @@ channels:
     - defaults
 dependencies:
     - cuml=23.06
-    - dgl=1.0.2
+    - dgl=1.1.1

examples/llm/vdb_upload/run.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,7 +46,7 @@ def run():
 )
 @click.option(
     "--model_max_batch_size",
-    default=64,
+    default=256,
     type=click.IntRange(min=1),
     help="Max batch size to use for the model",
 )

morpheus/_lib/src/objects/table_info.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -300,7 +300,8 @@ std::optional<std::string> MutableTableInfo::ensure_sliceable_index()
         auto df_index = py_df.attr("index");
 
         // Check to see if we actually need the change
-        if (df_index.attr("is_unique").cast<bool>() && df_index.attr("is_monotonic").cast<bool>())
+        if (df_index.attr("is_unique").cast<bool>() && (df_index.attr("is_monotonic_increasing").cast<bool>() ||
+                                                        df_index.attr("is_monotonic_decreasing").cast<bool>()))
         {
             // Set the outputname to nullopt
             old_index_col_name = std::nullopt;

morpheus/stages/general/monitor_stage.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023, NVIDIA CORPORATION.
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -99,7 +99,7 @@ def accepted_types(self) -> typing.Tuple:
     def supports_cpp_node(self):
         return False
 
-    def on_start(self):
+    async def start_async(self):
         """
         Starts the pipeline stage's progress bar.
         """

morpheus/stages/output/http_server_sink_stage.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023, NVIDIA CORPORATION.
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -142,7 +142,7 @@ def supports_cpp_node(self):
         """Indicates whether or not this stage supports a C++ node."""
         return False
 
-    def on_start(self):
+    async def start_async(self):
         """Starts the HTTP server."""
         from morpheus.common import HttpServer
         self._server = HttpServer(parse_fn=self._request_handler,