Merge pull request #34 from janhq/feat/init_cortex_tensorrtllm

feat: Init code for cortex.tensorrtllm

cpp/build_nitro.sh

@@ -1,4 +1,4 @@
 mkdir build
 cd build
-cmake .. -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_NITRO=ON -DBUILD_BATCH_MANAGER_DEFAULT=OFF -DCMAKE_CUDA_ARCHITECTURES=89-real -DTRT_LIB_DIR=/usr/local/tensorrt/lib -DTRT_INCLUDE_DIR=/usr/local/tensorrt/include -DCMAKE_BUILD_TYPE=Release
+cmake .. -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_NITRO=ON -DBUILD_CORTEX=ON -DBUILD_BATCH_MANAGER_DEFAULT=OFF -DCMAKE_CUDA_ARCHITECTURES=89-real -DTRT_LIB_DIR=/usr/local/tensorrt/lib -DTRT_INCLUDE_DIR=/usr/local/tensorrt/include -DCMAKE_BUILD_TYPE=Release
 make -j $(nproc)

cpp/tensorrt_llm/CMakeLists.txt

@@ -308,3 +308,7 @@ add_subdirectory(plugins)
 if(BUILD_NITRO)
   add_subdirectory(nitro)
 endif()
+
+if(BUILD_CORTEX)
+  add_subdirectory(cortex.tensorrtllm)
+endif()

cpp/tensorrt_llm/cortex.tensorrtllm/CMakeLists.txt

@@ -0,0 +1,129 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION &
+# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+# C++17
+# NItro init
+cmake_minimum_required(VERSION 3.5)
+project(cortex.tensorrtllm)
+set(TARGET engine)
+
+if(UNIX AND NOT APPLE)
+  set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
+  add_compile_options(-fPIC)
+endif()
+
+if(UNIX AND NOT APPLE)
+  add_compile_options(-fPIC)
+endif()
+
+include(CheckIncludeFileCXX)
+check_include_file_cxx(any HAS_ANY)
+check_include_file_cxx(string_view HAS_STRING_VIEW)
+check_include_file_cxx(coroutine HAS_COROUTINE)
+if(HAS_ANY
+   AND HAS_STRING_VIEW
+   AND HAS_COROUTINE)
+  set(CMAKE_CXX_STANDARD 20)
+elseif(HAS_ANY AND HAS_STRING_VIEW)
+  set(CMAKE_CXX_STANDARD 17)
+else()
+  set(CMAKE_CXX_STANDARD 14)
+endif()
+
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+SET(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+set(OPENSSL_USE_STATIC_LIBS TRUE)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+set(THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/build_deps/_install)
+
+message(STATUS "Current Cmake third party path: ${THIRD_PARTY_PATH}")
+
+find_library(JSONCPP
+    NAMES jsoncpp
+    HINTS "${THIRD_PARTY_PATH}/lib"
+)
+
+find_library(TRANTOR
+    NAMES trantor
+    HINTS "${THIRD_PARTY_PATH}/lib"
+)
+
+if(NOT WIN32) # Linux
+    # # Use pkg-config to find the SentencePiece library
+    # find_package(PkgConfig REQUIRED)
+    # # pkg_check_modules(SENTENCEPIECE REQUIRED sentencepiece)
+    # pkg_search_module(SENTENCEPIECE REQUIRED sentencepiece)
+    find_library(SENTENCEPIECE
+        NAMES sentencepiece
+        HINTS "${THIRD_PARTY_PATH}/lib"
+    )
+else() # Windows
+    set(SENTENCEPIECE_INCLUDE_DIRS "${THIRD_PARTY_PATH}/include")
+    set(SENTENCEPIECE_LIBRARY_DIRS "${THIRD_PARTY_PATH}/lib")
+endif()
+
+message(STATUS "SentencePiece library dirs: ${SENTENCEPIECE_LIBRARY_DIRS}")
+message(STATUS "SentencePiece header dirs: ${SENTENCEPIECE_INCLUDE_DIRS}")
+
+include_directories(${PROJECT_SOURCE_DIR}/include ${SENTENCEPIECE_INCLUDE_DIRS})
+
+link_directories(${SENTENCEPIECE_LIBRARY_DIRS})
+
+set(TOP_LEVEL_DIR "${PROJECT_SOURCE_DIR}/../../..")
+
+# add_custom_target(nitro_proj)
+# add_custom_target(cortex-tensorllm_proj)
+
+# !!!WARNING!!! Un-comment these CXXOPTS related lines after removing nitro
+# set(CXXOPTS_SRC_DIR ${TOP_LEVEL_DIR}/3rdparty/cxxopts)
+# add_subdirectory(${CXXOPTS_SRC_DIR} ${CMAKE_CURRENT_BINARY_DIR}/cxxopts)
+
+set(PLUGINS_SRC_DIR ${TOP_LEVEL_DIR}/cpp/build/tensorrt_llm/plugins)
+link_directories(${PLUGINS_SRC_DIR})
+
+set(TENSORRT_LLM_LIB ${TOP_LEVEL_DIR}/cpp/include/)
+
+# main
+add_library(${TARGET} SHARED 
+    src/process_manager.cc
+    src/tensorrtllm_engine.cc
+)
+
+target_link_libraries(
+    ${TARGET} PUBLIC ${SHARED_TARGET} nvinfer_plugin_tensorrt_llm cxxopts::cxxopts ${SENTENCEPIECE} PRIVATE ${JSONCPP} ${TRANTOR} ${CMAKE_THREAD_LIBS_INIT} )
+
+target_compile_features(${TARGET} PRIVATE cxx_std_17)
+target_compile_definitions(${TARGET} PUBLIC TOP_LEVEL_DIR="${TOP_LEVEL_DIR}")
+
+# aux_source_directory(controllers CTL_SRC)
+# aux_source_directory(common COMMON_SRC)
+# aux_source_directory(context CONTEXT_SRC)
+# aux_source_directory(models MODEL_SRC)
+# target_sources(nitro PRIVATE ${CTL_SRC} ${COMMON_SRC} ${CONTEXT_SRC})
+
+target_include_directories(${TARGET} PRIVATE 
+                           ${CMAKE_CURRENT_SOURCE_DIR} 
+                           ${CMAKE_CURRENT_SOURCE_DIR}/src/ 
+                           ${THIRD_PARTY_PATH}/include/
+                           ${TENSORRT_LLM_LIB})
+# ${CMAKE_CURRENT_SOURCE_DIR}/models)
+
+
+# add_dependencies(nitro_proj nitro)
+# add_dependencies(cortex-tensorllm_proj cortex-tensorllm)
+

cpp/tensorrt_llm/cortex.tensorrtllm/Makefile

@@ -0,0 +1,17 @@
+# Makefile for Cortex python-runtime engine - Build, Lint, Test, and Clean
+
+CMAKE_EXTRA_FLAGS ?= ""
+RUN_TESTS ?= true
+PYTHON_FILE_EXECUTION_PATH ?= .github/scripts/python-file-to-test.py
+
+# Default target, does nothing
+all:
+	@echo "Specify a target to run"
+
+# Build the Cortex python-runtime engine
+install-dependencies:
+ifeq ($(OS),Windows_NT) # Windows
+	cmd /C install_deps.bat
+else  # Unix-like systems (Linux and MacOS)
+	bash ./install_deps.sh
+endif

cpp/tensorrt_llm/cortex.tensorrtllm/base/cortex-common/enginei.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+
+#include "json/value.h"
+
+class EngineI {
+ public: 
+  virtual ~EngineI() {}
+
+  virtual void HandleChatCompletion(
+      std::shared_ptr<Json::Value> jsonBody,
+      std::function<void(Json::Value&&, Json::Value&&)>&& callback) = 0;
+  virtual void LoadModel(
+      std::shared_ptr<Json::Value> jsonBody,
+      std::function<void(Json::Value&&, Json::Value&&)>&& callback) = 0;
+  virtual void Destroy(
+      std::shared_ptr<Json::Value> jsonBody,
+      std::function<void(Json::Value&&, Json::Value&&)>&& callback) = 0;  
+};