Merge remote-tracking branch 'upstream/master' into golgeek/chat_template

Author: Ludovic LEROUX

.github/workflows/image_build.yml

@@ -280,6 +280,7 @@ jobs:
         run: |
           docker pull localai/localai:${{ steps.meta.outputs.version }}
           docker tag localai/localai:${{ steps.meta.outputs.version }} localai/localai:${{ inputs.latest-image }}
+          docker push localai/localai:${{ inputs.latest-image }}
           docker pull quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }}
           docker tag quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }} quay.io/go-skynet/local-ai:${{ inputs.latest-image }}
           docker push quay.io/go-skynet/local-ai:${{ inputs.latest-image }}
@@ -289,6 +290,7 @@ jobs:
         run: |
           docker pull localai/localai:${{ steps.meta_aio_dockerhub.outputs.version }}
           docker tag localai/localai:${{ steps.meta_aio_dockerhub.outputs.version }} localai/localai:${{ inputs.latest-image-aio }}
+          docker push localai/localai:${{ inputs.latest-image-aio }}
           docker pull quay.io/go-skynet/local-ai:${{ steps.meta_aio.outputs.version }}
           docker tag quay.io/go-skynet/local-ai:${{ steps.meta_aio.outputs.version }} quay.io/go-skynet/local-ai:${{ inputs.latest-image-aio }}
           docker push quay.io/go-skynet/local-ai:${{ inputs.latest-image-aio }}

Makefile

@@ -5,7 +5,7 @@ BINARY_NAME=local-ai
 
 # llama.cpp versions
 GOLLAMA_STABLE_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be
-CPPLLAMA_VERSION?=cc4a95426d17417d3c83f12bdb514fbe8abe2a88
+CPPLLAMA_VERSION?=1b67731e184e27a465b8c5476061294a4af668ea
 
 # gpt4all version
 GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all
@@ -16,7 +16,7 @@ RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp
 RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6
 
 # whisper.cpp version
-WHISPER_CPP_VERSION?=13c22321d1ac758ce68a429c23104e234b440769
+WHISPER_CPP_VERSION?=8f253ef3af1c62c04316ba4afa7145fc4d701a8c
 
 # bert.cpp version
 BERT_VERSION?=6abe312cded14042f6b7c3cd8edf082713334a4d

backend/python/autogptq/autogptq.py

@@ -39,7 +39,6 @@ def LoadModel(self, request, context):
                 self.model_name = "Qwen-VL-Chat"
                 model = AutoModelForCausalLM.from_pretrained(model_path, 
                     trust_remote_code=request.TrustRemoteCode,
-                    use_triton=request.UseTriton,
                     device_map="auto").eval()
             else:
                 model = AutoGPTQForCausalLM.from_quantized(model_path,

backend/python/common-env/transformers/transformers-nvidia.yml

@@ -116,7 +116,7 @@ dependencies:
       - sudachipy
       - sudachidict_core
       - vocos
-      - vllm==0.3.2
+      - vllm>=0.4.0
       - transformers>=4.38.2  # Updated Version
       - transformers_stream_generator==0.0.5
       - xformers==0.0.23.post1  

backend/python/common-env/transformers/transformers-rocm.yml

@@ -104,7 +104,7 @@ dependencies:
       - sudachipy
       - sudachidict_core
       - vocos
-      - vllm==0.3.2
+      - vllm>=0.4.0
       - transformers>=4.38.2  # Updated Version
       - transformers_stream_generator==0.0.5
       - xformers==0.0.23.post1

backend/python/common-env/transformers/transformers.yml

@@ -108,7 +108,7 @@ dependencies:
       - sudachipy
       - sudachidict_core
       - vocos
-      - vllm==0.3.2
+      - vllm>=0.4.0
       - transformers>=4.38.2  # Updated Version
       - transformers_stream_generator==0.0.5
       - xformers==0.0.23.post1  

docs/content/docs/features/gpt-vision.md

@@ -22,6 +22,17 @@ curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/jso
      "messages": [{"role": "user", "content": [{"type":"text", "text": "What is in the image?"}, {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" }}], "temperature": 0.9}]}'
 ```
 
+Grammars and function tools can be used as well in conjunction with vision APIs:
+
+```bash
+ curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+     "model": "llava", "grammar": "root ::= (\"yes\" | \"no\")",
+     "messages": [{"role": "user", "content": [{"type":"text", "text": "Is there some grass in the image?"}, {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" }}], "temperature": 0.9}]}'
+```
+
 ### Setup
 
+All-in-One images have already shipped the llava model as `gpt-4-vision-preview`, so no setup is needed in this case. 
+
 To setup the LLaVa models, follow the full example in the [configuration examples](https://github.com/mudler/LocalAI/blob/master/examples/configurations/README.md#llava).
+

docs/content/docs/features/openai-functions.md

@@ -144,6 +144,15 @@ curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/jso
    }'
 ```
 
+Grammars and function tools can be used as well in conjunction with vision APIs:
+
+```bash
+ curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+     "model": "llava", "grammar": "root ::= (\"yes\" | \"no\")",
+     "messages": [{"role": "user", "content": [{"type":"text", "text": "Is there some grass in the image?"}, {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" }}], "temperature": 0.9}]}'
+```
+
+
 ## 💡 Examples
 
 A full e2e example with `docker-compose` is available [here](https://github.com/go-skynet/LocalAI/tree/master/examples/functions).

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v2.11.0"
+  "version": "v2.12.1"
 }