From b716bf6344020a155a63e448caad4d5b6a469450 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Thu, 23 Jan 2025 18:36:29 -0800
Subject: [PATCH 01/29] Update run-readme-pr-macos.yml

source test commands instead of executing them.
(Possible fix for #1315 )
---
 .github/workflows/run-readme-pr-macos.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-readme-pr-macos.yml b/.github/workflows/run-readme-pr-macos.yml
index 64afe2247..852430320 100644
--- a/.github/workflows/run-readme-pr-macos.yml
+++ b/.github/workflows/run-readme-pr-macos.yml
@@ -33,7 +33,7 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs readme
+          . .ci/scripts/run-docs readme
   
           echo "::group::Completion"
           echo "tests complete"
@@ -68,7 +68,7 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs quantization
+          . .ci/scripts/run-docs quantization
   
           echo "::group::Completion"
           echo "tests complete"
@@ -103,7 +103,7 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs gguf
+          . .ci/scripts/run-docs gguf
 
           echo "::group::Completion"
           echo "tests complete"
@@ -137,7 +137,7 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs advanced
+          . .ci/scripts/run-docs advanced
 
           echo "::group::Completion"
           echo "tests complete"

From b0deb2ac81c41cc020b455dd756c0e924733e98b Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Thu, 23 Jan 2025 18:38:02 -0800
Subject: [PATCH 02/29] Update run-docs

source instead of exec
---
 .ci/scripts/run-docs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.ci/scripts/run-docs b/.ci/scripts/run-docs
index 521cfa811..b32168b38 100755
--- a/.ci/scripts/run-docs
+++ b/.ci/scripts/run-docs
@@ -17,7 +17,7 @@ if [ "$1" == "readme" ]; then
         echo "*******************************************"
         cat ./run-readme.sh
         echo "*******************************************"
-        bash -x ./run-readme.sh
+        . ./run-readme.sh
         echo "::endgroup::"
 
         exit 0
@@ -35,7 +35,7 @@ if [ "$1" == "quantization" ]; then
         echo "*******************************************"
         cat ./run-quantization.sh
         echo "*******************************************"
-        bash -x ./run-quantization.sh
+        . ./run-quantization.sh
         echo "::endgroup::"
 
         exit 0
@@ -53,7 +53,7 @@ if [ "$1" == "gguf" ]; then
         echo "*******************************************"
         cat ./run-gguf.sh
         echo "*******************************************"
-        bash -x ./run-gguf.sh
+        . ./run-gguf.sh
         echo "::endgroup::"
 fi
 
@@ -70,7 +70,7 @@ if [ "$1" == "advanced" ]; then
         echo "*******************************************"
         cat ./run-advanced.sh
         echo "*******************************************"
-        bash -x ./run-advanced.sh
+        . ./run-advanced.sh
         echo "::endgroup::"
 fi
 
@@ -86,7 +86,7 @@ if [ "$1" == "evaluation" ]; then
         echo "*******************************************"
         cat ./run-evaluation.sh
         echo "*******************************************"
-        bash -x ./run-evaluation.sh
+        . ./run-evaluation.sh
 fi
 
 if [ "$1" == "multimodal" ]; then
@@ -105,7 +105,7 @@ if [ "$1" == "multimodal" ]; then
         echo "*******************************************"
         cat ./run-multimodal.sh
         echo "*******************************************"
-        bash -x ./run-multimodal.sh
+        . ./run-multimodal.sh
         echo "::endgroup::"
 fi
 
@@ -122,7 +122,7 @@ if [ "$1" == "native" ]; then
         echo "*******************************************"
         cat ./run-native.sh
         echo "*******************************************"
-        bash -x ./run-native.sh
+        . ./run-native.sh
         echo "::endgroup::"
 fi
 
@@ -139,6 +139,6 @@ if [ "$1" == "distributed" ]; then
         echo "*******************************************"
         cat ./run-distributed.sh
         echo "*******************************************"
-        bash -x ./run-distributed.sh
+        . ./run-distributed.sh
         echo "::endgroup::"
 fi

From b7af6b99faee81d77022182922a7e8cd5abb814a Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Thu, 23 Jan 2025 19:14:36 -0800
Subject: [PATCH 03/29] Update README.md

somebody pushed all the model exports into exportedModels, but... we never create the directory.

we should do that also do this in the user instructions, just because storing into a directory that doesn't exist is not good :)
---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2448b0b72..36b343249 100644
--- a/README.md
+++ b/README.md
@@ -90,10 +90,11 @@ cd torchchat
 python3 -m venv .venv
 source .venv/bin/activate
 ./install/install_requirements.sh
+mkdir exportedModels
 ```
 [skip default]: end
 
-[shell default]: ./install/install_requirements.sh
+[shell default]: mkdir exportedModels; ./install/install_requirements.sh
 
 ## Commands
 

From 6ea3d55ba57e6b04e244360cb5ed4d78621cfca4 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 12:52:01 -0800
Subject: [PATCH 04/29] Update multimodal.md

multimodal doc needed end of tests comment.
---
 docs/multimodal.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/multimodal.md b/docs/multimodal.md
index cd249a1fb..975cdbd25 100644
--- a/docs/multimodal.md
+++ b/docs/multimodal.md
@@ -111,3 +111,5 @@ One of the goals of torchchat is to support various execution modes for every mo
 - **[ExecuTorch](https://github.com/pytorch/executorch)**: On-device (Edge) inference
 
 In addition, we are in the process of integrating with [lm_evaluation_harness](https://github.com/EleutherAI/lm-evaluation-harness) for multimodal model evaluation.
+
+[end default]: end

From 2e6b5ae7e9978e00f6ee973d9dd7b445193feb79 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 15:10:08 -0800
Subject: [PATCH 05/29] Update ADVANCED-USERS.md

Need to download files before using them, lol. We expect the users to do this, but we should verbalize.  Plus, if we extract for testing, then it obviously fails.
---
 docs/ADVANCED-USERS.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/ADVANCED-USERS.md b/docs/ADVANCED-USERS.md
index 17958e790..bd86d92be 100644
--- a/docs/ADVANCED-USERS.md
+++ b/docs/ADVANCED-USERS.md
@@ -185,6 +185,16 @@ MODEL_OUT=~/torchchat-exports
 
 mkdir -p ${MODEL_DIR}
 mkdir -p ${MODEL_OUT}
+
+# Change to the MODELDIR directory
+pushd ${MODEL_DIR}
+
+# Download the files for stories15M using wget
+wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt
+wget https://github.com/karpathy/llama2.c/raw/refs/heads/master/tokenizer.model
+
+# Go back to the original directory
+popd
 ```
 
 When we export models with AOT Inductor for servers and desktops, and
@@ -335,7 +345,7 @@ tests against the exported model with the same interface, and support
 additional experiments to confirm model quality and speed.
 
 ```
-python3 torchchat.py generate --device [ cuda | cpu ] --dso-path ${MODEL_NAME}.so --prompt "Once upon a time"
+python3 torchchat.py generate --device [ cuda | cpu ] --checkpoint-path ${MODEL_PATH} --dso-path ${MODEL_NAME}.so --prompt "Once upon a time"
 ```
 
 

From 52fd00ba701012f52a2e7d4e9dc9c47c296284cd Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 16:45:02 -0800
Subject: [PATCH 06/29] Update native-execution.md

( triggers unexpected token in macos zsh
---
 docs/native-execution.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/native-execution.md b/docs/native-execution.md
index c22d3c3ba..dc0c799b1 100644
--- a/docs/native-execution.md
+++ b/docs/native-execution.md
@@ -83,6 +83,7 @@ python3 torchchat.py export stories15M --output-dso-path ./model.so
 We can now execute the runner with:
 
 [shell default]: pip install wget
+
 ```
 curl -OL https://github.com/karpathy/llama2.c/raw/master/tokenizer.model
 ./cmake-out/aoti_run ./model.so -z ./tokenizer.model -l 2 -i "Once upon a time"
@@ -109,7 +110,7 @@ installed ExecuTorch, running the commands below will build the
 runner, without re-installing ExecuTorch from source:
 
 ```
-# Pull submodules (re2, abseil) for Tiktoken
+# Pull submodules re2 and abseil for Tiktoken
 git submodule sync
 git submodule update --init
 

From 76f7edf69283843af95efef193ec3f97c2557b4b Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 16:46:41 -0800
Subject: [PATCH 07/29] Update run-readme-pr-macos.yml

          # metadata does not install properly on macos
          # .ci/scripts/run-docs multimodal
---
 .github/workflows/run-readme-pr-macos.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-macos.yml b/.github/workflows/run-readme-pr-macos.yml
index 51255a025..a12e10894 100644
--- a/.github/workflows/run-readme-pr-macos.yml
+++ b/.github/workflows/run-readme-pr-macos.yml
@@ -213,7 +213,8 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs multimodal
+          # metadata does not install properly on macos
+          # .ci/scripts/run-docs multimodal
 
           echo "::group::Completion"
           echo "tests complete"

From da9a92aac789506ac8cf60cb8ec428ca5fae8c95 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 16:47:28 -0800
Subject: [PATCH 08/29] Update run-readme-pr-mps.yml

          # metadata does not install properly on macos
          # .ci/scripts/run-docs multimodal
---
 .github/workflows/run-readme-pr-mps.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-mps.yml b/.github/workflows/run-readme-pr-mps.yml
index 4d5cd7e14..04acc9c20 100644
--- a/.github/workflows/run-readme-pr-mps.yml
+++ b/.github/workflows/run-readme-pr-mps.yml
@@ -161,7 +161,8 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs multimodal
+          # metadata does not install properly on macos
+          # .ci/scripts/run-docs multimodal
 
           echo "::group::Completion"
           echo "tests complete"

From 3e4ad3dee4f3a73b0fa472187a23234694a156d5 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 17:16:41 -0800
Subject: [PATCH 09/29] Update ADVANCED-USERS.md

install wget
---
 docs/ADVANCED-USERS.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/ADVANCED-USERS.md b/docs/ADVANCED-USERS.md
index bd86d92be..9e006acf2 100644
--- a/docs/ADVANCED-USERS.md
+++ b/docs/ADVANCED-USERS.md
@@ -177,6 +177,8 @@ preparatory step:
 You can set these variables as follows for the exemplary model15M
 model from Andrej Karpathy's tinyllamas model family:
 
+[shell default]: pip install wget
+
 ```
 MODEL_NAME=stories15M
 MODEL_DIR=~/checkpoints/${MODEL_NAME}

From c2cb227fa84d6f9bad70ab7026f0a7746110fb8f Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 17:20:15 -0800
Subject: [PATCH 10/29] Update run-readme-pr-macos.yml

          echo ".ci/scripts/run-docs native DISABLED"
          # .ci/scripts/run-docs native
---
 .github/workflows/run-readme-pr-macos.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-macos.yml b/.github/workflows/run-readme-pr-macos.yml
index a12e10894..6a1fab3e5 100644
--- a/.github/workflows/run-readme-pr-macos.yml
+++ b/.github/workflows/run-readme-pr-macos.yml
@@ -248,7 +248,8 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs native
+          echo ".ci/scripts/run-docs native DISABLED"
+          # .ci/scripts/run-docs native
 
           echo "::group::Completion"
           echo "tests complete"

From 72702f00a646a7bd3af6aa1a48d5db7c5c6b37e3 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 24 Jan 2025 17:21:01 -0800
Subject: [PATCH 11/29] Update run-readme-pr-mps.yml

          echo ".ci/scripts/run-docs native DISABLED"
          # .ci/scripts/run-docs native
---
 .github/workflows/run-readme-pr-mps.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-mps.yml b/.github/workflows/run-readme-pr-mps.yml
index 04acc9c20..e4ecd0b12 100644
--- a/.github/workflows/run-readme-pr-mps.yml
+++ b/.github/workflows/run-readme-pr-mps.yml
@@ -189,7 +189,8 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs native
+          echo ".ci/scripts/run-docs native DISABLED"
+          # .ci/scripts/run-docs native
 
           echo "::group::Completion"
           echo "tests complete"

From 79c4a232dacf6a67cb17755f716e51a6c40d99c6 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Tue, 28 Jan 2025 15:58:02 -0800
Subject: [PATCH 12/29] Update run-docs

switch to gs=32 quantization
(requires consolidated run-docs of #1439)
---
 .ci/scripts/run-docs | 202 +++++++++++++------------------------------
 1 file changed, 62 insertions(+), 140 deletions(-)

diff --git a/.ci/scripts/run-docs b/.ci/scripts/run-docs
index 017ed69b9..2c0dfdf76 100755
--- a/.ci/scripts/run-docs
+++ b/.ci/scripts/run-docs
@@ -1,145 +1,67 @@
-# /bin/bash -x
+#!/bin/bash -x
 
-if [ "X$1" == "X" ]; then
+# Check if an argument was provided
+if [ -z "$1" ]; then
   echo "Must specify document to run"
   exit 1
 fi
 
-if [ "$1" == "readme" ]; then
-        echo "::group::Create script to run README"
-        python3 torchchat/utils/scripts/updown.py --create-sections --file README.md --replace 'llama3.1:stories15M,-l 3:-l 2' --suppress huggingface-cli,HF_TOKEN > ./run-readme.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-readme.sh
-        echo "::endgroup::"
-
-        echo "::group::Run README"
-        echo "*******************************************"
-        cat ./run-readme.sh
-        echo "*******************************************"
-        . ./run-readme.sh
-        echo "::endgroup::"
-
-        exit 0
-fi
-
-if [ "$1" == "quantization" ]; then
-        echo "::group::Create script to run quantization"
-        python3 torchchat/utils/scripts/updown.py --create-sections --file docs/quantization.md --replace llama3:stories15M --suppress huggingface-cli,HF_TOKEN > ./run-quantization.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-quantization.sh
-        echo "::endgroup::"
-
-        echo "::group::Run quantization"
-        echo "*******************************************"
-        cat ./run-quantization.sh
-        echo "*******************************************"
-        . ./run-quantization.sh
-        echo "::endgroup::"
-
-        exit 0
-fi
-
-if [ "$1" == "gguf" ]; then
-        echo "::group::Create script to run gguf"
-        python3 torchchat/utils/scripts/updown.py --file docs/GGUF.md --replace 'llama3:stories15M,-l 3:-l 2' --suppress huggingface-cli,HF_TOKEN > ./run-gguf.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-gguf.sh
-        echo "::endgroup::"
-
-        echo "::group::Run gguf"
-        echo "*******************************************"
-        cat ./run-gguf.sh
-        echo "*******************************************"
-        . ./run-gguf.sh
-        echo "::endgroup::"
-fi
-
-
-if [ "$1" == "advanced" ]; then
-        echo "::group::Create script to run advanced"
-        python3 torchchat/utils/scripts/updown.py --file docs/ADVANCED-USERS.md --replace 'llama3:stories15M,-l 3:-l 2' --suppress huggingface-cli,HF_TOKEN > ./run-advanced.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-advanced.sh
-        echo "::endgroup::"
-
-        echo "::group::Run advanced"
-        echo "*******************************************"
-        cat ./run-advanced.sh
-        echo "*******************************************"
-        . ./run-advanced.sh
-        echo "::endgroup::"
-fi
-
-if [ "$1" == "evaluation" ]; then
-        echo "::group::Create script to run evaluation"
-        python3 torchchat/utils/scripts/updown.py --file torchchat/utils/docs/evaluation.md --replace 'llama3:stories15M,-l 3:-l 2' --suppress huggingface-cli,HF_TOKEN > ./run-evaluation.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-evaluation.sh
-        echo "::endgroup::"
-
-        echo "::group::Run evaluation"
-        echo "*******************************************"
-        cat ./run-evaluation.sh
-        echo "*******************************************"
-        . ./run-evaluation.sh
-fi
-
-if [ "$1" == "multimodal" ]; then
-
-   # Expecting that this might fail this test as-is, because 
-   # it's the first on-pr test depending on github secrets for access with HF token access
-
-        echo "::group::Create script to run multimodal"
-        python3 torchchat/utils/scripts/updown.py --file docs/multimodal.md > ./run-multimodal.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-multimodal.sh
-        echo "::endgroup::"
-
-        echo "::group::Run multimodal"
-        echo "*******************************************"
-        cat ./run-multimodal.sh
-        echo "*******************************************"
-        . ./run-multimodal.sh
-        echo "::endgroup::"
-fi
-
-if [ "$1" == "native" ]; then
-
-        echo "::group::Create script to run native-execution"
-        python3 torchchat/utils/scripts/updown.py --file docs/native-execution.md > ./run-native.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-native.sh
-        echo "::endgroup::"
-
-        echo "::group::Run native-execution"
-        echo "*******************************************"
-        cat ./run-native.sh
-        echo "*******************************************"
-        . ./run-native.sh
-        echo "::endgroup::"
-fi
-
-if [ "$1" == "distributed" ]; then
-
-        echo "::group::Create script to run distributed"
-        python3 torchchat/utils/scripts/updown.py --file docs/distributed.md --replace 'llama3.1:stories110M,-l 3:-l 2'  --suppress huggingface-cli,HF_TOKEN  > ./run-distributed.sh
-        python3 torchchat/utils/scripts/updown.py --file docs/distributed.md --suppress huggingface-cli,HF_TOKEN > ./run-distributed.sh
-        # for good measure, if something happened to updown processor,
-        # and it did not error out, fail with an exit 1
-        echo "exit 1" >> ./run-distributed.sh
-        echo "::endgroup::"
-
-        echo "::group::Run distributed"
-        echo "*******************************************"
-        cat ./run-distributed.sh
-        echo "*******************************************"
-        . ./run-distributed.sh
-        echo "::endgroup::"
-fi
+# Pre-initialize variables
+filepath=""
+parameters="--replace 'llama3:stories15M,-l3:-l2,mobile.json:mobile-32.json,cuda.json:cuda-32.json' --suppress huggingface-cli,HF_TOKEN"
+script_name="./run-${1}.sh"  # Dynamically initialize script name
+
+# Use a case statement to handle the $1 argument
+case "$1" in
+  "readme")
+    filepath="README.md"
+    ;;
+  "quantization")
+    filepath="docs/quantization.md"
+    ;;
+  "gguf")
+    filepath="docs/GGUF.md"
+    ;;
+  "advanced")
+    filepath="docs/ADVANCED-USERS.md"
+    ;;
+  "evaluation")
+    filepath="torchchat/utils/docs/evaluation.md"
+    ;;
+  "multimodal")
+    filepath="docs/multimodal.md"
+    parameters=""  # Clear parameters
+    ;;
+  "native")
+    filepath="docs/native-execution.md"
+    parameters=""  # Clear parameters
+    ;;
+  "distributed")
+    filepath="docs/distributed.md"
+    parameters="--replace 'llama3.1:stories110M,-l3:-l2' --suppress huggingface-cli,HF_TOKEN"  # Use stories110M to avoid need for authentication
+    ;;
+  "local")
+    filepath="docs/local-model.md"
+    parameters=""  # Clear parameters
+    ;;
+
+  *)
+    echo "Unknown option: $1"
+    exit 1
+    ;;
+esac
+
+# Generate the script
+echo "::group::Create script to run $1"
+python3 torchchat/utils/scripts/updown.py --file "$filepath" $parameters > "$script_name"
+# if something happened to updown processor, and it did not error out, fail with an exit 1
+echo "exit 1" >> "$script_name"
+echo "::endgroup::"
+
+# Run the script
+echo "::group::Run $1"
+echo "*******************************************"
+cat "$script_name"
+echo "*******************************************"
+. "$script_name"
+echo "::endgroup::"

From ed702afe321e710a564162f7bae18322bd09e877 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Tue, 28 Jan 2025 15:58:59 -0800
Subject: [PATCH 13/29] Create cuda-32.json

add gs=32 cuda quantization for use w/ stories15M
---
 torchchat/quant_config/cuda-32.json | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 torchchat/quant_config/cuda-32.json

diff --git a/torchchat/quant_config/cuda-32.json b/torchchat/quant_config/cuda-32.json
new file mode 100644
index 000000000..90c37250a
--- /dev/null
+++ b/torchchat/quant_config/cuda-32.json
@@ -0,0 +1,5 @@
+{
+    "executor": {"accelerator": "cuda"},
+    "precision": {"dtype": "bf16"},
+    "linear:int4": {"groupsize" : 32}
+}

From 286bb081a32909b4a2e08c55b1921a30128e1ee2 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Tue, 28 Jan 2025 15:59:34 -0800
Subject: [PATCH 14/29] Create mobile-32.json

add gs=32 for stories15M
---
 torchchat/quant_config/mobile-32.json | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 torchchat/quant_config/mobile-32.json

diff --git a/torchchat/quant_config/mobile-32.json b/torchchat/quant_config/mobile-32.json
new file mode 100644
index 000000000..3afaa7542
--- /dev/null
+++ b/torchchat/quant_config/mobile-32.json
@@ -0,0 +1,4 @@
+{
+    "embedding": {"bitwidth": 4, "groupsize" : 32},
+    "linear:a8w4dq": {"groupsize" : 32}
+}

From 0e21e956a457117179625514745f3e5acb7974fe Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Thu, 30 Jan 2025 15:14:51 -0800
Subject: [PATCH 15/29] Update run-readme-pr.yml

Comment out tests that currently fail, as per summary in PR comments
---
 .github/workflows/run-readme-pr.yml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml
index 37c27822b..48f8f346f 100644
--- a/.github/workflows/run-readme-pr.yml
+++ b/.github/workflows/run-readme-pr.yml
@@ -106,7 +106,8 @@ jobs:
         uname -a
         echo "::endgroup::"
 
-        .ci/scripts/run-docs gguf
+        # failing
+        # .ci/scripts/run-docs gguf
 
         echo "::group::Completion"
         echo "tests complete"
@@ -128,7 +129,8 @@ jobs:
         uname -a
         echo "::endgroup::"
 
-        TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs gguf
+        # failing
+        # TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs gguf
 
         echo "::group::Completion"
         echo "tests complete"
@@ -151,7 +153,8 @@ jobs:
         uname -a
         echo "::endgroup::"
 
-        .ci/scripts/run-docs advanced
+        # failing
+        # .ci/scripts/run-docs advanced
 
         echo "::group::Completion"
         echo "tests complete"
@@ -174,7 +177,8 @@ jobs:
         uname -a
         echo "::endgroup::"
 
-        TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs advanced
+        # failing
+        # TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs advanced
 
         echo "::group::Completion"
         echo "tests complete"
@@ -196,7 +200,7 @@ jobs:
         uname -a
         echo "::endgroup::"
 
-        .ci/scripts/run-docs evaluation
+        # .ci/scripts/run-docs evaluation
 
         echo "::group::Completion"
         echo "tests complete"
@@ -218,7 +222,7 @@ jobs:
         uname -a
         echo "::endgroup::"
 
-        TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs evaluation
+        # TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs evaluation
 
         echo "::group::Completion"
         echo "tests complete"

From e901c036e5aae93445e07e9cd53acafea3855525 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Thu, 30 Jan 2025 23:58:13 -0800
Subject: [PATCH 16/29] Update install_requirements.sh

Dump location of executable to understand these errors:
https://hud.pytorch.org/pr/pytorch/torchchat/1476#36452260294

2025-01-31T00:18:57.1405698Z + pip3 install -r install/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
2025-01-31T00:18:57.1406689Z ./install/install_requirements.sh: line 101: pip3: command not found
---
 install/install_requirements.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/install/install_requirements.sh b/install/install_requirements.sh
index 360ba1801..56010c218 100755
--- a/install/install_requirements.sh
+++ b/install/install_requirements.sh
@@ -19,6 +19,7 @@ then
   fi
 fi
 echo "Using python executable: $PYTHON_EXECUTABLE"
+echo "located at $(which $PYTHON_EXECUTABLE || echo not found)"
 
 PYTHON_SYS_VERSION="$($PYTHON_EXECUTABLE -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")"
 # Check python version. Expect at least 3.10.x
@@ -43,6 +44,7 @@ else
 fi
 
 echo "Using pip executable: $PIP_EXECUTABLE"
+echo "located at $(which $PIP_EXECUTABLE || echo not found)"
 
 # Since torchchat often uses main-branch features of pytorch, only the nightly
 # pip versions will have the required features. The PYTORCH_NIGHTLY_VERSION value should

From 684816a42c03f80b4679a1cacfeaa3bb98faab18 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 00:00:48 -0800
Subject: [PATCH 17/29] Update install_requirements.sh

dump candidate locations for pip
---
 install/install_requirements.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/install/install_requirements.sh b/install/install_requirements.sh
index 56010c218..9736bb7d6 100755
--- a/install/install_requirements.sh
+++ b/install/install_requirements.sh
@@ -46,6 +46,13 @@ fi
 echo "Using pip executable: $PIP_EXECUTABLE"
 echo "located at $(which $PIP_EXECUTABLE || echo not found)"
 
+echo
+echo "possible pip candidates are:"
+echo "pip is located at $(which pip || echo not found)"
+echo "pip3 is located at $(which pip3 || echo not found)"
+echo "pip{PYTHON_SYS_VERSION} is located at $(which pip{PYTHON_SYS_VERSION} || echo not found)"
+echo
+
 # Since torchchat often uses main-branch features of pytorch, only the nightly
 # pip versions will have the required features. The PYTORCH_NIGHTLY_VERSION value should
 # agree with the third-party/pytorch pinned submodule commit.

From b3c4b9e1c712ad4350558a5b2b37ec733d15a555 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 00:13:25 -0800
Subject: [PATCH 18/29] Update README.md

Some of the updown commands were getting rendered. Not sure why/when that happens?
---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index f13c0e758..cfc96b774 100644
--- a/README.md
+++ b/README.md
@@ -239,7 +239,9 @@ python3 torchchat.py server llama3.1
 ```
 [skip default]: end
 
+<!==
 [shell default]: python3 torchchat.py server llama3.1 & server_pid=$! ; sleep 90 # wait for server to be ready to accept requests
+-->
 
 In another terminal, query the server using `curl`. Depending on the model configuration, this query might take a few minutes to respond.
 
@@ -280,7 +282,9 @@ curl http://127.0.0.1:5000/v1/chat/completions \
 
 [skip default]: end
 
+<!--
 [shell default]: kill ${server_pid}
+-->
 
 </details>
 

From ead5b6a043f05a29b5fc9b8e1dc86943be738353 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 00:15:11 -0800
Subject: [PATCH 19/29] Update run-docs

readme switched from llama3 to llama3.1, so replace llama3.1 with stories15M
---
 .ci/scripts/run-docs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/scripts/run-docs b/.ci/scripts/run-docs
index 69c69b672..42ae11fe8 100755
--- a/.ci/scripts/run-docs
+++ b/.ci/scripts/run-docs
@@ -17,6 +17,7 @@ script_name="./run-${1}.sh"  # Dynamically initialize script name
 case "$1" in
   "readme")
     filepath="README.md"
+    parameters="--replace 'llama3.1:stories15M,-l3:-l2,mobile.json:mobile-32.json' --suppress huggingface-cli,HF_TOKEN"
     ;;
   "quantization")
     filepath="docs/quantization.md"

From 835ae0e0648ddcc67a46ea68f24a2334cb3a8101 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 00:22:06 -0800
Subject: [PATCH 20/29] Update run-readme-pr-macos.yml

remove failing gguf test
---
 .github/workflows/run-readme-pr-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-macos.yml b/.github/workflows/run-readme-pr-macos.yml
index 6a1fab3e5..16da5fc31 100644
--- a/.github/workflows/run-readme-pr-macos.yml
+++ b/.github/workflows/run-readme-pr-macos.yml
@@ -109,7 +109,7 @@ jobs:
 
           echo "using workaround for #1416 and #1315 by setting torchchat device explicitly"
           export TORCHCHAT_DEVICE=cpu 
-          .ci/scripts/run-docs gguf
+          # .ci/scripts/run-docs gguf
 
           echo "::group::Completion"
           echo "tests complete"

From 30f6ba813115d97b326bcc002931e47824e8b522 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 00:22:38 -0800
Subject: [PATCH 21/29] Update run-readme-pr-mps.yml

Remove failing gguf test
---
 .github/workflows/run-readme-pr-mps.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-mps.yml b/.github/workflows/run-readme-pr-mps.yml
index 8657ab9f8..80b836e2b 100644
--- a/.github/workflows/run-readme-pr-mps.yml
+++ b/.github/workflows/run-readme-pr-mps.yml
@@ -81,7 +81,7 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
-          .ci/scripts/run-docs gguf
+          # .ci/scripts/run-docs gguf
 
           echo "::group::Completion"
           echo "tests complete"

From 5e651266e70bbefe219833bc13d4096a04e4e2e7 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 13:14:08 -0800
Subject: [PATCH 22/29] Update run-readme-pr.yml

Can we mix `steps:` with `script: |` in git workflows?

Testing 123 testing!
---
 .github/workflows/run-readme-pr.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml
index 48f8f346f..d9baf3989 100644
--- a/.github/workflows/run-readme-pr.yml
+++ b/.github/workflows/run-readme-pr.yml
@@ -18,6 +18,13 @@ jobs:
       gpu-arch-type: cuda
       gpu-arch-version: "12.4"
       timeout: 60
+      steps:
+        - name: Checkout repo
+          uses: actions/checkout@v3
+        - name: Setup Python
+          uses: actions/setup-python@v4
+          with:
+            python-version: '3.10.11'
       script: |
         echo "::group::Print machine info"
         uname -a

From 8519a44960888873f5a08afedf9660dc91800be3 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 14:44:46 -0800
Subject: [PATCH 23/29] Update run-docs

remove quotes around replace as the nested quotes are not interpreted by the shall but seem to be passed to updown.py.

We don't have spaces in replace, so no need for escapes.
---
 .ci/scripts/run-docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/scripts/run-docs b/.ci/scripts/run-docs
index 42ae11fe8..55af099f6 100755
--- a/.ci/scripts/run-docs
+++ b/.ci/scripts/run-docs
@@ -10,7 +10,7 @@ fi
 filepath=""
 # cuda supports padding, so no need to replace quantization for now.  
 # otherwise add: 'cuda.json:cuda-32.json' to replace rules
-parameters="--replace 'llama3:stories15M,-l3:-l2,mobile.json:mobile-32.json' --suppress huggingface-cli,HF_TOKEN"
+parameters="--replace llama3:stories15M,-l3:-l2,mobile.json:mobile-32.json --suppress huggingface-cli,HF_TOKEN"
 script_name="./run-${1}.sh"  # Dynamically initialize script name
 
 # Use a case statement to handle the $1 argument

From d5b360712f75d0ca34a5d90b028357efd80cd11e Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 14:51:42 -0800
Subject: [PATCH 24/29] Update run-readme-pr.yml

1 - Remove steps experiment.
2 - add at-get install pip3

Maybe releng needs to look at what's happening with pip?
---
 .github/workflows/run-readme-pr.yml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml
index d9baf3989..9a92f8c48 100644
--- a/.github/workflows/run-readme-pr.yml
+++ b/.github/workflows/run-readme-pr.yml
@@ -19,13 +19,10 @@ jobs:
       gpu-arch-version: "12.4"
       timeout: 60
       steps:
-        - name: Checkout repo
-          uses: actions/checkout@v3
-        - name: Setup Python
-          uses: actions/setup-python@v4
-          with:
-            python-version: '3.10.11'
       script: |
+        set -x
+        apt-get install pip3 pip || true
+        
         echo "::group::Print machine info"
         uname -a
         echo "::endgroup::"

From f15bc156c6f6edf67f1983b848037a1fd4e987e8 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 17:41:14 -0800
Subject: [PATCH 25/29] Update run-docs

remove quotes that mess up parameter identification.
---
 .ci/scripts/run-docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/scripts/run-docs b/.ci/scripts/run-docs
index 55af099f6..4e5881c42 100755
--- a/.ci/scripts/run-docs
+++ b/.ci/scripts/run-docs
@@ -17,7 +17,7 @@ script_name="./run-${1}.sh"  # Dynamically initialize script name
 case "$1" in
   "readme")
     filepath="README.md"
-    parameters="--replace 'llama3.1:stories15M,-l3:-l2,mobile.json:mobile-32.json' --suppress huggingface-cli,HF_TOKEN"
+    parameters="--replace llama3.1:stories15M,-l3:-l2,mobile.json:mobile-32.json --suppress huggingface-cli,HF_TOKEN"
     ;;
   "quantization")
     filepath="docs/quantization.md"

From 2a18f0dbf54c739778ef543f2311386330b1ed2b Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Fri, 31 Jan 2025 17:44:09 -0800
Subject: [PATCH 26/29] Update run-readme-pr.yml

try to install pip & pip3
---
 .github/workflows/run-readme-pr.yml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml
index 9a92f8c48..4e7e2c17e 100644
--- a/.github/workflows/run-readme-pr.yml
+++ b/.github/workflows/run-readme-pr.yml
@@ -18,12 +18,10 @@ jobs:
       gpu-arch-type: cuda
       gpu-arch-version: "12.4"
       timeout: 60
-      steps:
       script: |
+        echo "::group::Print machine info and try install pip and/or pip3"
         set -x
         apt-get install pip3 pip || true
-        
-        echo "::group::Print machine info"
         uname -a
         echo "::endgroup::"
 
@@ -45,7 +43,9 @@ jobs:
       gpu-arch-version: "12.4"
       timeout: 60
       script: |
-        echo "::group::Print machine info"
+        echo "::group::Print machine info and try install pip and/or pip3"
+        set -x
+        apt-get install pip3 pip || true
         uname -a
         echo "::endgroup::"
 
@@ -67,7 +67,9 @@ jobs:
       gpu-arch-version: "12.4"
       timeout: 60
       script: |
-        echo "::group::Print machine info"
+        echo "::group::Print machine info and try install pip and/or pip3"
+        set -x
+        apt-get install pip3 pip || true
         uname -a
         echo "::endgroup::"
 
@@ -89,7 +91,9 @@ jobs:
       gpu-arch-version: "12.4"
       timeout: 60
       script: |
-        echo "::group::Print machine info"
+        echo "::group::Print machine info and try install pip and/or pip3"
+        set -x
+        apt-get install pip3 pip || true
         uname -a
         echo "::endgroup::"
 

From 30746fce11967def4e33bc74c44274bfb38644f7 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Sat, 1 Feb 2025 22:03:43 -0800
Subject: [PATCH 27/29] Update run-readme-pr.yml

debug

        which pip || true
        which pip3 || true
        which conda || true
---
 .github/workflows/run-readme-pr.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/run-readme-pr.yml b/.github/workflows/run-readme-pr.yml
index 4e7e2c17e..a3504d19c 100644
--- a/.github/workflows/run-readme-pr.yml
+++ b/.github/workflows/run-readme-pr.yml
@@ -21,10 +21,19 @@ jobs:
       script: |
         echo "::group::Print machine info and try install pip and/or pip3"
         set -x
+        which pip || true
+        which pip3 || true
+        which conda || true
         apt-get install pip3 pip || true
+        which pip || true
+        which pip3 || true
+        which conda || true
         uname -a
         echo "::endgroup::"
 
+        which pip || true
+        which pip3 || true
+        which conda || true
         .ci/scripts/run-docs readme
 
         echo "::group::Completion"
@@ -47,6 +56,9 @@ jobs:
         set -x
         apt-get install pip3 pip || true
         uname -a
+        which pip || true
+        which pip3 || true
+        which conda || true
         echo "::endgroup::"
 
         TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs readme

From fb4e0ddb439aa82395ea6282fc61ae7b2a70a1cb Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Sat, 1 Feb 2025 22:04:34 -0800
Subject: [PATCH 28/29] Update run-readme-pr-macos.yml

---
 .github/workflows/run-readme-pr-macos.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/run-readme-pr-macos.yml b/.github/workflows/run-readme-pr-macos.yml
index 16da5fc31..114d0a569 100644
--- a/.github/workflows/run-readme-pr-macos.yml
+++ b/.github/workflows/run-readme-pr-macos.yml
@@ -33,6 +33,10 @@ jobs:
           sysctl machdep.cpu.core_count
           echo "::endgroup::"
 
+          which pip || true
+          which pip3 || true
+          which conda || true
+
           echo "using workaround for #1416 and #1315 by setting torchchat device explicitly"
           export TORCHCHAT_DEVICE=cpu 
           . .ci/scripts/run-docs readme

From 7786b84b6b586e98c9ea896f1234e51d82bd6b14 Mon Sep 17 00:00:00 2001
From: Michael Gschwind <61328285+mikekgfb@users.noreply.github.com>
Date: Sat, 1 Feb 2025 22:06:12 -0800
Subject: [PATCH 29/29] Update run-readme-pr-linuxaarch64.yml

debug info

```
        which pip || true
        which pip3 || true
        which conda || true
```
---
 .../workflows/run-readme-pr-linuxaarch64.yml  | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-readme-pr-linuxaarch64.yml b/.github/workflows/run-readme-pr-linuxaarch64.yml
index 1f22c4f2e..e765e1993 100644
--- a/.github/workflows/run-readme-pr-linuxaarch64.yml
+++ b/.github/workflows/run-readme-pr-linuxaarch64.yml
@@ -23,6 +23,9 @@ jobs:
         uname -a
         echo "::endgroup::"
 
+        which pip || true
+        which pip3 || true
+        which conda || true
         TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs readme
 
         echo "::group::Completion"
@@ -44,7 +47,11 @@ jobs:
         echo "::group::Print machine info"
         uname -a
         echo "::endgroup::"
-
+   
+        which pip || true
+        which pip3 || true
+        which conda || true
+        
         TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs quantization
 
   test-gguf-cpu:
@@ -62,6 +69,10 @@ jobs:
         uname -a
         echo "::endgroup::"
 
+        which pip || true
+        which pip3 || true
+        which conda || true
+        
         TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs gguf
 
         echo "::group::Completion"
@@ -84,6 +95,10 @@ jobs:
         uname -a
         echo "::endgroup::"
 
+        which pip || true
+        which pip3 || true
+        which conda || true
+        
         TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs advanced
 
         echo "::group::Completion"
@@ -106,6 +121,10 @@ jobs:
         uname -a
         echo "::endgroup::"
 
+        which pip || true
+        which pip3 || true
+        which conda || true
+        
         TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs evaluation
 
         echo "::group::Completion"