From 82812a691f0f1af73db492975be7b0de4920ddf0 Mon Sep 17 00:00:00 2001 From: Hien To Date: Mon, 4 Nov 2024 16:02:43 +0700 Subject: [PATCH] docs: installation linux instruction --- docs/docs/installation.mdx | 81 ++++++++++++++ docs/docs/installation/docker.mdx | 21 ++-- docs/docs/installation/linux.mdx | 173 ++++++++++++++---------------- docs/sidebars.ts | 6 +- 4 files changed, 175 insertions(+), 106 deletions(-) create mode 100644 docs/docs/installation.mdx diff --git a/docs/docs/installation.mdx b/docs/docs/installation.mdx new file mode 100644 index 000000000..ec59350ee --- /dev/null +++ b/docs/docs/installation.mdx @@ -0,0 +1,81 @@ +--- +title: Installation +description: Cortex Installation +slug: "installation" +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import Admonition from '@theme/Admonition'; + +:::warning +🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. +::: + +## Cortex.cpp Installation +### Cortex.cpp offers four installer types +- Network Installers download a minimal system and require an internet connection to fetch packages during installation. +- Local Installers include all necessary packages, enabling offline installation without internet access. +- Dockerfile Installers are used to build a Docker image for Cortex. +- Binary files without package management. + +### Cortex.cpp supports three channels +- Stable: The latest stable release on github. +- Beta: The release candidate for the next stable release, available on github release with the tag `vx.y.z-rc1` +- Nightly: The nightly build of the latest code on dev branch, available on [discord](https://discord.com/channels/1107178041848909847/1283654073488379904). + +### Download URLs + +| Type | Version | Operating System | Download Link | +|----------|---------------|------------------|---------------------------------------------------------------------------------------------------| +| **Local** | **Stable** | **Windows** | [Download](https://app.cortexcpp.com/download/latest/windows-amd64-local) | +| **Local** | **Stable** | **Linux** | [Download](https://app.cortexcpp.com/download/latest/linux-amd64-local) | +| **Local** | **Stable** | **MacOS** | [Download](https://app.cortexcpp.com/download/latest/mac-universal-local) | + +:::info +For other versions, please look at [cortex.cpp repo](https://github.com/janhq/cortex.cpp) or each installer page. +::: + + +## Minimum Requirements + +### OS +- MacOS 12 or later +- Windows 10 or later +- Linux: Ubuntu 20.04 or later, Debian 11 or later (For other distributions, please use the Dockerfile installer or binary files, we have not tested on other distributions yet.) + +### Hardware +#### CPU +:::info +- Cortex.cpp supports a processor that can handle AVX2. For the full list, please see [here](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#CPUs_with_AVX2). +- We support older processors with AVX, AVX-512 and non-AVX, though this is not recommended. +::: + +##### Intel CPU +- Haswell processors (Q2 2013) and newer. +- Tiger Lake (Q3 2020) and newer for Celeron and Pentium processors. +##### AMD CPU +- Excavator processors (Q2 2015) and newer. + +#### RAM +:::info +We support DDR2 RAM as the minimum requirement but recommend using newer generations of RAM for improved performance. +::: +- 8GB for running up to 3B models (int4). +- 16GB for running up to 7B models (int4). +- 32GB for running up to 13B models (int4). + +#### GPU +:::info +Having at least 6GB VRAM when using NVIDIA, AMD, or Intel Arc GPUs is recommended. +::: +- 6GB can load the 3B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU. +- 8GB can load the 7B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU. +- 12GB can load the 13B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU. +:::info +- [NVIDIA driver](https://www.nvidia.com/Download/index.aspx) version 470.63.01 or higher. + *Note: Users must install the driver by themselves.* +- [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) version 12.0 or higher. *Note: Cortex.cpp will automatically detect and install the required version of cudart to the user's machine.* +::: +#### Disk +- At least 10GB for app storage and model download. \ No newline at end of file diff --git a/docs/docs/installation/docker.mdx b/docs/docs/installation/docker.mdx index 02ec55076..e9bc2e725 100644 --- a/docs/docs/installation/docker.mdx +++ b/docs/docs/installation/docker.mdx @@ -1,23 +1,26 @@ - --- title: Docker description: Install Cortex using Docker. --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import Admonition from '@theme/Admonition'; + :::warning 🚧 **Cortex.cpp is currently in development.** The documentation describes the intended functionality, which may not yet be fully implemented. ::: -# Setting Up Cortex with Docker +## Setting Up Cortex with Docker This guide walks you through the setup and running of Cortex using Docker. -## Prerequisites +### Prerequisites - Docker or Docker Desktop - `nvidia-container-toolkit` (for GPU support) -## Setup Instructions +### Setup Instructions 1. **Clone the Cortex Repository** ```bash @@ -64,11 +67,11 @@ This guide walks you through the setup and running of Cortex using Docker. cortex --help ``` -## Usage +### Usage With Docker running, you can use the following commands to interact with Cortex. Ensure the container is running and `curl` is installed on your machine. -### 1. List Available Engines +#### 1. List Available Engines ```bash curl --request GET --url http://localhost:39281/v1/engines --header "Content-Type: application/json" @@ -98,7 +101,7 @@ curl --request GET --url http://localhost:39281/v1/engines --header "Content-Typ } ``` -### 2. Pull Models from Hugging Face +#### 2. Pull Models from Hugging Face - Open a terminal and run `websocat ws://localhost:39281/events` to capture download events, follow [this instruction](https://github.com/vi/websocat?tab=readme-ov-file#installation) to install `websocat`. - In another terminal, pull models using the commands below. @@ -118,7 +121,7 @@ curl --request GET --url http://localhost:39281/v1/engines --header "Content-Typ curl --request GET --url http://localhost:39281/v1/models ``` -### 3. Start a Model and Send an Inference Request +#### 3. Start a Model and Send an Inference Request - **Start the model:** ```bash @@ -140,7 +143,7 @@ curl --request GET --url http://localhost:39281/v1/engines --header "Content-Typ }' ``` -### 4. Stop a Model +#### 4. Stop a Model - To stop a running model, use: ```bash diff --git a/docs/docs/installation/linux.mdx b/docs/docs/installation/linux.mdx index ce074f753..e32a65fab 100644 --- a/docs/docs/installation/linux.mdx +++ b/docs/docs/installation/linux.mdx @@ -13,119 +13,102 @@ import Admonition from '@theme/Admonition'; ::: ## Cortex.cpp Installation -Cortex.cpp offers two installer types: -- Network Installers download a minimal system and require an internet connection to fetch packages during installation. -- Local Installers include all necessary packages, enabling offline installation without internet access. :::info -Before installation, make sure that you have met the required [dependencies](#dependencies) and [hardware](#hardware) to run Cortex. +Before installation, make sure that you have met the [minimum requirements](/docs/installation#minimum-requirements) to run Cortex. +This instruction is for stable releases. For beta and nightly releases, please replace `cortex` with `cortex-beta` and `cortex-nightly`, respectively. ::: -1. Download the Linux installer: - - `.deb`: - - Stable: https://github.com/janhq/cortex.cpp/releases - - Beta: https://github.com/janhq/cortex.cpp/releases - - Nightly: https://github.com/janhq/cortex.cpp/releases - - `.appImage`: - - Stable: https://github.com/janhq/cortex.cpp/releases - - Beta: https://github.com/janhq/cortex.cpp/releases - - Nightly: https://github.com/janhq/cortex.cpp/releases - -2. Ensure that Cortex.cpp is sucessfulyy installed: -```bash -# Stable -cortex -# Beta -cortex-beta +### Prerequisites +- OpenMPI -# Nightly -cortex-nightly -``` +### Install Cortex.cpp + +1. Download the Linux installer: + - From release: https://github.com/janhq/cortex.cpp/releases + - From quick download links: + - Local installer `.deb`: + - Stable: https://app.cortexcpp.com/download/latest/linux-amd64-local + - Beta: https://app.cortexcpp.com/download/beta/linux-amd64-local + - Nightly: https://app.cortexcpp.com/download/nightly/linux-amd64-local + - Network installer `.deb`: + - Stable: https://app.cortexcpp.com/download/latest/linux-amd64-network + - Beta: https://app.cortexcpp.com/download/beta/linux-amd64-network + - Nightly: https://app.cortexcpp.com/download/nightly/linux-amd64-network + - Binary: + - Stable: https://app.cortexcpp.com/download/latest/linux-amd64-binary + - Beta: https://app.cortexcpp.com/download/beta/linux-amd64-binary + - Nightly: https://app.cortexcpp.com/download/nightly/linux-amd64-binary + +2. Install Cortex.cpp using the following command: + ```bash + # Installer + sudo apt install ./cortex--linux-amd64-network-installer.deb + + + # Binary + tar -xvf cortex--linux-amd64.tar.gz + cd cortex + sudo mv cortex /usr/bin/cortex + sudo chmod +x /usr/bin/Cortexs + sudo mv cortex-server /usr/bin/cortex-server + + ## For binary, you need to install engine manually after extracting the binary + cortex engines install llama-cpp + ``` + +3. Ensure that Cortex.cpp is sucessfulyy installed: + ```bash + # Stable + cortex -v + ``` ### Data Folder By default, Cortex.cpp is installed in the following directory: ``` # Binary Location -/usr/bin/cortexcpp +/usr/bin/cortex +/usr/bin/cortex-server # Application Data (Engines, Models and Logs folders) -/home//.cortexcpp +/home//cortexcpp + +# Configuration File +/home//.cortexrc ``` + ## Uninstall Cortex.cpp ```bash # Stable version sudo apt remove cortexcpp - -# Beta version -sudo apt remove cortexcpp-beta - -# Nightly version -sudo apt remove cortexcpp-nightly ``` ## Build from Source -1. Clone the Cortex.cpp repository [here](https://github.com/janhq/cortex.cpp). -2. Navigate to the `engine > vcpkg` folder. -3. Configure the vpkg: -```bash -cd vcpkg -./bootstrap-vcpkg.sh -vcpkg install -``` -4. Build the Cortex.cpp inside the `build` folder: - -```bash -mkdir build -cd build -cmake .. -DCMAKE_TOOLCHAIN_FILE=path_to_vcpkg_folder/vcpkg/scripts/buildsystems/vcpkg.cmake -make -j4 -``` -5. Use Visual Studio with the C++ development kit to build the project using the files generated in the `build` folder. -6. Verify that Cortex.cpp is installed correctly by getting help information. - -```sh -# Get the help information -cortex -h -``` - -## Prerequisites -### Dependencies -- Node.js version 18 and higher -- NPM -- Homebrew +### Prerequisites - OpenMPI - -### Hardware -#### Operating System -- Debian-based (Supports `.deb` and `AppImage` ) -- Ubuntu version 22.04 and higher -#### CPU -:::info -- Cortex.cpp supports a processor that can handle AVX2. For the full list, please see [here](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#CPUs_with_AVX2). -- We support older processors with AVX and AVX-512, though this is not recommended. -::: -##### Intel CPU -- Haswell processors (Q2 2013) and newer. -- Tiger Lake (Q3 2020) and newer for Celeron and Pentium processors. -##### AMD CPU -- Excavator processors (Q2 2015) and newer. -#### RAM -:::info -We support DDR2 RAM as the minimum requirement but recommend using newer generations of RAM for improved performance. -::: -- 8GB for running up to 3B models (int4). -- 16GB for running up to 7B models (int4). -- 32GB for running up to 13B models (int4). -#### GPU -:::info -Having at least 6GB VRAM when using NVIDIA, AMD, or Intel Arc GPUs is recommended. -::: -- 6GB can load the 3B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU. -- 8GB can load the 7B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU. -- 12GB can load the 13B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU. -:::info -- [NVIDIA driver](https://www.nvidia.com/Download/index.aspx) version 470.63.01 or higher. -- [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) version 12.3 or higher. -::: -#### Disk -- At least 10GB for app storage and model download. \ No newline at end of file +- CMake >= 3.10 +- gcc/g++ >= 9 +- ninja-build + +### Build Cortex.cpp + +1. **Clone the Cortex Repository** + ```bash + git clone https://github.com/janhq/cortex.cpp.git + cd cortex.cpp + git submodule update --init + ``` +2. Build the Cortex.cpp : + + ```bash + cd engine + make configure-vcpkg + make build CMAKE_EXTRA_FLAGS="-DCORTEX_CPP_VERSION=$(git rev-parse HEAD) -DCMAKE_BUILD_TEST=OFF -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake + ``` + +3. Verify that Cortex.cpp is builded correctly by getting help information. + + ```sh + # Get the help information + ./build/cortex -h + ``` diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 36a3e7420..b057cd9f8 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -35,7 +35,8 @@ const sidebars: SidebarsConfig = { type: "category", label: "Installation", link: { - type: "generated-index", + type: "doc", + id: "installation" }, collapsed: true, items: [ @@ -68,7 +69,8 @@ const sidebars: SidebarsConfig = { type: "category", label: "Architecture", link: { - type: "generated-index", + type: "doc", + id: "architecture" }, collapsed: true, items: [