Skip to content

Commit

Permalink
add the rest of TheBloke's quantized llama2-7b-chat models
Browse files Browse the repository at this point in the history
  • Loading branch information
@cameronfyfe
cameronfyfe committed Feb 19, 2024
1 parent 416a6af commit e37457a
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 25 deletions.
84 changes: 60 additions & 24 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,28 +37,64 @@ nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llam
# Full Command List

```present ./scripts/list-cmds.sh github:cameronfyfe/nix-run-ai
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_L_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q6_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q8_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_L_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q6_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q8_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_L_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q6_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q8_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-13B-chat-GGUF__llama-2-13b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_L_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q3_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_0_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_S_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q6_K_gguf
nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q8_0_gguf
```
9 changes: 9 additions & 0 deletions models.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,17 @@
commit = "191239b3e26b2882fb562ffccdd1cf0f65402adb";
models = {
"llama-2-7b-chat.Q2_K.gguf" = "sha256-wN0wTXYejgXQgswpAtdiSn+HhY/fqk7wmDMP/nZ/8NM=";
"llama-2-7b-chat.Q3_K_L.gguf" = "sha256-3bPHhY2JEovcu0QdZjZRAGG0ur4kmNOrhlJTln4UFwc=";
"llama-2-7b-chat.Q3_K_M.gguf" = "sha256-h+D0c8SRoVNcjuhKETzS6g7bpZ2L8PHDCom1MZ53RR4=";
"llama-2-7b-chat.Q3_K_S.gguf" = "sha256-RpOPLBtvgXXNk/vI8RpRUYSixSx6mNaIwdxvt46gAsI=";
"llama-2-7b-chat.Q4_0.gguf" = "sha256-mVjum2cFlBR7dQu8fQVAuSj6EtzF3UxYzFbtLrheNxs=";
"llama-2-7b-chat.Q4_K_M.gguf" = "sha256-CKVWbWHXy2tCDD5Dh6OeAHjh8v5fBV86A4hzhTBNS/o=";
"llama-2-7b-chat.Q4_K_S.gguf" = "sha256-Yy+nX5S0aWDePK9iPnmx0KTIVks5pfqsu4wf29m6P1I=";
"llama-2-7b-chat.Q5_0.gguf" = "sha256-DVXEEzlk+A7jGZeFPLg2N648wlhji3/q6dGqVgaole4=";
"llama-2-7b-chat.Q5_K_M.gguf" = "sha256-4LmZIM9HuUx40vsGoezrnteVF236P3/qxkYp8bUrmX8=";
"llama-2-7b-chat.Q5_K_S.gguf" = "sha256-EXGKeXm9dhUUpCRS3C8SP2GRsXbgUa6znseRxbTs+l8=";
"llama-2-7b-chat.Q6_K.gguf" = "sha256-zt2CXGFVdbzVjrl8rmXslyr9F7ABKfNhtrw/NjPc0B0=";
"llama-2-7b-chat.Q8_0.gguf" = "sha256-9H2t5ehkZu22bFr+b46fsfuywpKCe5C9RrehgX2GS/I=";
};
};
"Llama-2-13B-chat-GGUF" = {
Expand Down
3 changes: 2 additions & 1 deletion scripts/list-cmds.sh
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

FLAKE=${1:-'.'}
shift

OS=$(uname -s | tr 'A-Z' 'a-z')
ARCH=$(uname -m | sed 's/arm64/aarch64/')
Expand All @@ -10,5 +11,5 @@ PLATFORM="$ARCH-$OS"
PACKAGES=$(nix eval .#packages.$PLATFORM --apply builtins.attrNames | tr -d '[]"')
for PACKAGE in $PACKAGES; do
echo "nix run $FLAKE#$PACKAGE"
echo "nix run $FLAKE#$PACKAGE $@"
done

0 comments on commit e37457a

Please sign in to comment.