-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Pouya Rostam
committed
May 7, 2024
1 parent
e3f9b01
commit 569346a
Showing
12 changed files
with
1,133 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
name: C Codestyle | ||
|
||
on: | ||
workflow_dispatch: | ||
push: | ||
branches: [ master ] | ||
paths: | ||
- '**/*.c' | ||
- '.github/workflows/c-codestyle.yml' | ||
pull_request: | ||
branches: [ master, 'v[0-9]+.[0-9]+' ] | ||
paths: | ||
- '**/*.c' | ||
- '.github/workflows/c-codestyle.yml' | ||
|
||
jobs: | ||
check-c-codestyle: | ||
runs-on: ubuntu-latest | ||
|
||
steps: | ||
- uses: actions/checkout@v3 | ||
|
||
- name: Set up Python 3.10 | ||
uses: actions/setup-python@v4 | ||
with: | ||
python-version: '3.10' | ||
|
||
- name: Install dependencies | ||
run: sudo apt install clang-format | ||
|
||
- name: Check c codestyle | ||
run: python3 resources/.lint/c/formatter.py -c -v |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
name: C Demos | ||
|
||
on: | ||
workflow_dispatch: | ||
push: | ||
branches: [ main ] | ||
paths: | ||
- '!demo/c/README.md' | ||
- '.github/workflows/c-demos.yml' | ||
- 'demo/c/**' | ||
- 'include/**' | ||
- 'lib/common/**' | ||
- 'lib/jetson/**' | ||
- 'lib/linux/**' | ||
- 'lib/mac/**' | ||
- 'lib/raspberry-pi/**' | ||
- 'lib/windows/**' | ||
- 'resources/.test/**' | ||
pull_request: | ||
branches: [ main, 'v[0-9]+.[0-9]+' ] | ||
paths: | ||
- '!demo/c/README.md' | ||
- '.github/workflows/c-demos.yml' | ||
- 'demo/c/**' | ||
- 'include/**' | ||
- 'lib/common/**' | ||
- 'lib/jetson/**' | ||
- 'lib/linux/**' | ||
- 'lib/mac/**' | ||
- 'lib/raspberry-pi/**' | ||
- 'lib/windows/**' | ||
- 'resources/.test/**' | ||
|
||
defaults: | ||
run: | ||
working-directory: demo/c | ||
|
||
jobs: | ||
build-self-hosted: | ||
runs-on: ${{ matrix.machine }} | ||
|
||
strategy: | ||
fail-fast: false | ||
matrix: | ||
xpu: [ cpu ] | ||
make_file: [ "Unix Makefiles" ] | ||
machine: [ rpi4-32, rpi4-64, rpi5-32, rpi5-64, pv-linux, pv-ios ] | ||
include: | ||
- xpu: gpu | ||
machine: pv-linux | ||
make_file: "Unix Makefiles" | ||
- xpu: gpu | ||
machine: pv-windows | ||
make_file: "MinGW Makefiles" | ||
- xpu: cpu | ||
machine: pv-windows | ||
make_file: "MinGW Makefiles" | ||
|
||
steps: | ||
- uses: actions/checkout@v3 | ||
|
||
- name: Setup Python virtualenv (*nix) | ||
if: ${{ matrix.machine != 'pv-windows' }} | ||
run: | | ||
python3 -m venv binding/python/.venv | ||
. binding/python/.venv/bin/activate | ||
echo PATH=$PATH >> $GITHUB_ENV | ||
- name: Setup Python virtualenv (windows) | ||
if: ${{ matrix.machine == 'pv-windows' }} | ||
run: | | ||
python3 -m venv binding\python\.venv | ||
binding\python\.venv\Scripts\activate | ||
echo PATH=$PATH >> $GITHUB_ENV | ||
- name: Download resource files | ||
run: curl http://${{secrets.PV_CICD_RES_SERVER_AUTHORITY}}/github/picollm/res/phi2-290.bin/latest/phi2-290.bin -o phi2-290.bin | ||
|
||
- name: Create build directory | ||
run: cmake -G "${{ matrix.make_file }}" -B ./build | ||
|
||
- name: Build demo | ||
run: cmake --build ./build --target picollm_demo_completion | ||
|
||
- name: Install dependencies | ||
run: python3 -m pip install -r test/requirements.txt | ||
|
||
- name: Test | ||
run: python test/test_picollm_c.py ${{secrets.PV_VALID_ACCESS_KEY}} ./phi2-290.bin ${{matrix.xpu}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
cmake-build-debug | ||
build |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
cmake_minimum_required(VERSION 3.13) | ||
project(picollm_demo) | ||
|
||
set(CMAKE_C_STANDARD 99) | ||
set(CMAKE_BUILD_TYPE Release) | ||
|
||
include_directories("${PROJECT_SOURCE_DIR}/../../include") | ||
|
||
add_executable(picollm_demo_completion picollm_demo_completion.c) | ||
|
||
if (NOT WIN32) | ||
target_link_libraries(picollm_demo_completion dl) | ||
endif() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,150 @@ | ||
# picoLLM Inference Engine C Demo | ||
|
||
## picoLLM Inference Engine | ||
|
||
picoLLM Inference Engine is a highly accurate and cross-platform SDK optimized for running compressed large language | ||
models. picoLLM Inference Engine is: | ||
|
||
- Accurate; picoLLM Compression improves GPTQ by up to 98%. | ||
- Private; LLM inference runs 100% locally. | ||
- Cross-Platform | ||
- Runs on CPU and GPU | ||
- Free for open-weight models | ||
|
||
## Compatibility | ||
|
||
- C99-compatible compiler | ||
- Runs on Linux (x86_64), macOS (arm64, x86_64), Windows (x86_64), and Raspberry Pi (5, 4, and 3). | ||
|
||
## Requirements | ||
|
||
- [CMake](https://cmake.org/) version 3.13 or higher | ||
- [MinGW](https://mingw-w64.org/) (**Windows Only**) | ||
|
||
## AccessKey | ||
|
||
AccessKey is your authentication and authorization token for deploying Picovoice SDKs, including picoLLM. Anyone who is | ||
using Picovoice needs to have a valid AccessKey. You must keep your AccessKey secret. You would need internet | ||
connectivity to validate your AccessKey with Picovoice license servers even though the LLM inference is running 100% | ||
offline and completely free for open-weight models. Everyone who signs up for | ||
[Picovoice Console](https://console.picovoice.ai/) receives a unique AccessKey. | ||
|
||
## Usage | ||
|
||
The demo accepts a prompt and a set of optional parameters and generates a single completion. It can run all models, whether instruction-tuned or not. | ||
|
||
### Build Linux/MacOS | ||
|
||
Build the demo by running this from the root of the repository: | ||
|
||
```console | ||
cmake -S demo/c/ -B demo/c/build | ||
cmake --build demo/c/build | ||
``` | ||
|
||
### Build Windows | ||
|
||
Build the demo by running this from the root of the repository: | ||
|
||
```console | ||
cmake -S demo/c/ -B demo/c/build -G "MinGW Makefiles" | ||
cmake --build demo/c/build | ||
``` | ||
|
||
### Run | ||
|
||
Running the demo without arguments prints the usage: | ||
|
||
```console | ||
``` | ||
|
||
Run the command corresponding to your platform from the root of the repository. Replace `${ACCESS_KEY}` with yours obtained from Picovoice Console, `${MODEL_PATH}` with the path to a model file | ||
downloaded from Picovoice Console, and `${PROMPT}` with a prompt string. | ||
|
||
To get information about all the available options in the demo, run the following: | ||
|
||
```console | ||
picollm_demo_completion --help | ||
``` | ||
|
||
#### Linux (x86_64) | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/linux/x86_64/libpv_leopard.so \ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### macOS (x86_64) | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/mac/x86_64/libpv_leopard.dylib \ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### macOS (arm64) | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/mac/arm64/libpv_leopard.dylib \ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### Windows | ||
|
||
Run using `Command Prompt`. | ||
|
||
```console | ||
demo\\c\\build\\leopard_demo.exe ^ | ||
-a ${ACCESS_KEY} ^ | ||
-m lib\\common\\leopard_params.pv ^ | ||
-l lib\\windows\\amd64\\libpv_leopard.dll ^ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### Raspberry Pi 4 | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/raspberry-pi/cortex-a72/libpv_leopard.so \ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### Raspberry Pi 4 (64-bit) | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/raspberry-pi/cortex-a72-aarch64/libpv_leopard.so \ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### Raspberry Pi 3 | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/raspberry-pi/cortex-a53/libpv_leopard.so \ | ||
${AUDIO_PATH} | ||
``` | ||
|
||
#### Raspberry Pi 3 (64-bit) | ||
|
||
```console | ||
./demo/c/build/leopard_demo \ | ||
-a ${ACCESS_KEY} \ | ||
-m lib/common/leopard_params.pv \ | ||
-l lib/raspberry-pi/cortex-a53-aarch64/libpv_leopard.so \ | ||
${AUDIO_PATH} | ||
``` |
Oops, something went wrong.