initial commit

.github/workflows/repo-checks.yaml

@@ -0,0 +1,34 @@
+name: Repo Checks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  fmt-check:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - uses: actions/checkout@v4
+
+      - run: nix develop -c just fmt-check
+
+  readme-check:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - uses: actions/checkout@v4
+
+      - run: nix develop -c just readme-check
+

.gitignore

@@ -0,0 +1,3 @@
+/result
+/tmp/
+*.log

Justfile

@@ -0,0 +1,20 @@
+_:
+    @just --list
+
+fmt:
+    nixpkgs-fmt .
+
+fmt-check:
+    nixpkgs-fmt --check .
+
+# Update generated sections in readme
+readme-update:
+    present --in-place README.md
+
+# Check generated sections in readme
+readme-check: _tmp
+    present README.md > tmp/README.md
+    diff README.md tmp/README.md
+
+_tmp:
+    mkdir -p tmp

README.md

@@ -0,0 +1,60 @@
+# nix-run-ai
+
+Nix one liners for running machine learning models.
+
+# Requirements
+
+The only requirements are nix with flakes enabled and hardware capable of running a given model. As long as the underlying package supports a host system these commands should work on any Linux or MacOS system with any chip architecture.
+
+Install nix
+
+https://nixos.org/download.html
+
+Enable flakes
+
+    sudo sh -c 'echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf'
+
+# Examples
+
+Run llama.cpp on a GPU with CUDA using a local model
+
+```bash
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda -- \
+  --model <my-model> \
+  --n-gpu-layers 43 \
+  --main-gpu 1 \
+  --port 4000
+```
+
+Run llama.cpp on a CPU using a pre-loaded quantized llama2-7b model
+
+```bash
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf -- \
+  --threads 4 \
+  --port 4000
+```
+
+# Full Command List
+
+```present ./scripts/list-cmds.sh github:cameronfyfe/nix-run-ai
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__main__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cpu__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__CapybaraHermes-2_5-Mistral-7B-GGUF__capybarahermes-2_5-mistral-7b_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q2_K_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q4_K_M_gguf
+nix run github:cameronfyfe/nix-run-ai#llama-cpp__server__cuda__HF__TheBloke__Llama-2-7B-Chat-GGUF__llama-2-7b-chat_Q5_K_M_gguf
+```

flake.nix

@@ -0,0 +1,40 @@
+{
+  inputs = {
+    flake-utils.url = "github:numtide/flake-utils";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = inputs @ { self, ... }:
+    (inputs.flake-utils.lib.eachDefaultSystem (system:
+      let
+
+        pkgs = import inputs.nixpkgs {
+          inherit system;
+          config = {
+            allowUnfree = true;
+          };
+        };
+
+        inherit (pkgs) callPackage mkShell writeShellScriptBin;
+
+        models = callPackage ./models.nix { };
+
+        llama-cpp = callPackage ./llama-cpp.nix { inherit models; };
+
+      in
+      rec {
+
+        devShells = {
+          default = mkShell ({
+            buildInputs = with pkgs; [
+              just
+              nixpkgs-fmt
+              present-cli
+            ];
+          });
+        };
+
+        packages = llama-cpp.packages;
+
+      }));
+}

llama-cpp.nix

@@ -0,0 +1,65 @@
+{ writeShellScriptBin
+, llama-cpp
+, models
+}:
+
+let
+
+  inherit (builtins) attrNames filter foldl';
+
+  # llama-cpp commands, modes, and models to generate packages for
+  cmds = [ "main" "server" ];
+  modes = [ "cpu" "cuda" ];
+  llama-cpp-supported-models =
+    # TODO:
+    #   Put variables in `models` object to indicate what each model can run on
+    #   and use that info here to generate the right list.
+    #   For now all models in this repo are for llama-cpp so grab all of them
+    (filter
+      (s: s != "override" && s != "overrideDerivation")
+      (attrNames models)
+    )
+    # (attrNames models)
+    ++ [ null ] # this generates a package without an embedded model that can be passed models at runtime
+  ;
+
+  # package set with all combinations of llama-cpp commands, modes, and models
+  packages =
+    foldl'
+      (acc: cmd: acc //
+        (foldl'
+          (acc: mode: acc //
+            (foldl'
+              (acc: model: acc // (
+                let
+                  overrides = if mode == "cuda" then { cudaSupport = true; } else { };
+                  args = if model != null then "--model ${models.${model}}" else "";
+                  bin = "${llama-cpp.override overrides}/bin/llama-cpp-${cmd}";
+                  name =
+                    if model != null then
+                      "llama-cpp__${cmd}__${mode}__${model}" else
+                      "llama-cpp__${cmd}__${mode}";
+                in
+                {
+                  "${name}" = writeShellScriptBin "nix-run-llama-cpp" ''
+                    #!/usr/bin/env bash
+                    ${bin} ${args} $@
+                  '';
+                }
+              ))
+              { }
+              llama-cpp-supported-models
+            )
+          )
+          { }
+          modes
+        )
+      )
+      { }
+      cmds;
+
+in
+
+{
+  inherit packages;
+}

models.nix

@@ -0,0 +1,84 @@
+{ fetchurl
+}:
+
+let
+
+  inherit (builtins) attrNames foldl' replaceStrings;
+
+  fetchHuggingFaceModel = { owner, repo, commit, model, hash }:
+    fetchurl {
+      url = "https://huggingface.co/${owner}/${repo}/resolve/${commit}/${model}?download=true";
+      inherit hash;
+    };
+
+  models = {
+    # HuggingFace
+    HF = {
+      TheBloke = {
+        "CapybaraHermes-2.5-Mistral-7B-GGUF" = {
+          commit = "8bea614edd9a2d5d9985a6e6c1ecc166261cacb8";
+          models = {
+            "capybarahermes-2.5-mistral-7b.Q4_K_M.gguf" = "sha256-68zLv7A+UIiM14B0XdYcHz2gLTheeNvCxQYtzcWlycg=";
+          };
+        };
+        "Llama-2-7B-Chat-GGUF" = {
+          commit = "191239b3e26b2882fb562ffccdd1cf0f65402adb";
+          models = {
+            "llama-2-7b-chat.Q2_K.gguf" = "sha256-wN0wTXYejgXQgswpAtdiSn+HhY/fqk7wmDMP/nZ/8NM=";
+            "llama-2-7b-chat.Q4_K_M.gguf" = "sha256-CKVWbWHXy2tCDD5Dh6OeAHjh8v5fBV86A4hzhTBNS/o=";
+            "llama-2-7b-chat.Q5_K_M.gguf" = "sha256-4LmZIM9HuUx40vsGoezrnteVF236P3/qxkYp8bUrmX8=";
+          };
+        };
+      };
+    };
+  };
+
+  # # HuggingFace model list (objects with parameters for fetchHuggingFaceModel function)
+  hfModelList =
+    foldl'
+      (acc: owner: acc ++
+        (foldl'
+          (acc: repo: acc ++
+            (foldl'
+              (acc: model: acc ++ [
+                {
+                  inherit owner repo model;
+                  commit = models.HF.${owner}."${repo}".commit;
+                  hash = models.HF.${owner}."${repo}".models."${model}";
+                }
+              ])
+              [ ]
+              (attrNames models.HF.${owner}.${repo}.models)
+            )
+          )
+          [ ]
+          (attrNames models.HF.${owner})
+        )
+      )
+      [ ]
+      (attrNames models.HF);
+
+  # HuggingFace model packages
+  hfModelPackages =
+    foldl'
+      (acc: pkg: acc // (with pkg;
+      let
+        repoStr = replaceStrings [ "." ] [ "_" ] repo;
+        modelStr = replaceStrings [ "." ] [ "_" ] model;
+      in
+      {
+        "HF__${owner}__${repoStr}__${modelStr}" = fetchHuggingFaceModel {
+          inherit owner repo model;
+          commit = models.HF.${owner}.${repo}.commit;
+          hash = models.HF.${owner}.${repo}.models."${model}";
+        };
+      }
+      ))
+      { }
+      hfModelList;
+
+  packages = hfModelPackages;
+
+in
+
+packages

scripts/list-cmds.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+FLAKE=${1:-'.'}
+
+OS=$(uname -s | tr 'A-Z' 'a-z')
+ARCH=$(uname -m | sed 's/arm64/aarch64/')
+
+PLATFORM="$ARCH-$OS"
+
+PACKAGES=$(nix eval .#packages.$PLATFORM --apply builtins.attrNames | tr -d '[]"')
+
+for PACKAGE in $PACKAGES; do
+  echo "nix run $FLAKE#$PACKAGE"
+done