matlab-deep-learning · Oct 30, 2024 · Oct 31, 2024 · Nov 1, 2024 · Nov 4, 2024 · Nov 4, 2024
diff --git a/+llms/+internal/callAzureChatAPI.m b/+llms/+internal/callAzureChatAPI.m
@@ -73,8 +73,20 @@
     if isempty(nvp.StreamFun)
         message = response.Body.Data.choices(1).message;
     else
-        message = struct("role", "assistant", ...
-            "content", streamedText);
+        pat = '{"' + wildcardPattern + '":';
+        if contains(streamedText,pat)
+            s = jsondecode(streamedText);
+            if contains(s.function.arguments,pat)
+                prompt = jsondecode(s.function.arguments);
+                s.function.arguments = prompt;
+            end
+            message = struct("role", "assistant", ...
+                 "content",[], ...
+                 "tool_calls",jsondecode(streamedText));
+        else
+            message = struct("role", "assistant", ...
+                "content", streamedText);
+        end
     end
     if isfield(message, "tool_choice")
         text = "";

diff --git a/+llms/+internal/callOllamaChatAPI.m b/+llms/+internal/callOllamaChatAPI.m
@@ -1,4 +1,4 @@
-function [text, message, response] = callOllamaChatAPI(model, messages, nvp)
+function [text, message, response] = callOllamaChatAPI(model, messages, functions, nvp)
 % This function is undocumented and will change in a future release
 
 %callOllamaChatAPI Calls the Ollama™ chat completions API.
@@ -22,11 +22,13 @@
 %   % Send a request
 %   [text, message] = llms.internal.callOllamaChatAPI(model, messages)
 
-%   Copyright 2023-2024 The MathWorks, Inc.
+%   Copyright 2023-2025 The MathWorks, Inc.
 
 arguments
     model
     messages
+    functions
+    nvp.ToolChoice
     nvp.Temperature
     nvp.TopP
     nvp.MinP
@@ -52,7 +54,7 @@
     nvp.StopSequences = [nvp.StopSequences, nvp.StopSequences];
 end
 
-parameters = buildParametersCall(model, messages, nvp);
+parameters = buildParametersCall(model, messages, functions, nvp);
 
 [response, streamedText] = llms.internal.sendRequestWrapper(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
 
@@ -61,7 +63,11 @@
 if response.StatusCode=="OK"
     % Outputs the first generation
     if isempty(nvp.StreamFun)
-        message = response.Body.Data.message;
+        if iscell(response.Body.Data)
+            message = response.Body.Data{1}.message;
+        else
+            message = response.Body.Data.message;
+        end
     else
         message = struct("role", "assistant", ...
             "content", streamedText);
@@ -73,7 +79,7 @@
 end
 end
 
-function parameters = buildParametersCall(model, messages, nvp)
+function parameters = buildParametersCall(model, messages, functions, nvp)
 % Builds a struct in the format that is expected by the API, combining
 % MESSAGES, FUNCTIONS and parameters in NVP.
 
@@ -83,7 +89,24 @@
 
 parameters.stream = ~isempty(nvp.StreamFun);
 
+if ~isempty(functions)
+    parameters.tools = functions;
+end
+
+if ~isempty(nvp.ToolChoice)
+    parameters.tool_choice = nvp.ToolChoice;
+end
+
 options = struct;
+
+if strcmp(nvp.ResponseFormat,"json")
+    parameters.format = struct('type','json_object');
+elseif isstruct(nvp.ResponseFormat)
+    parameters.format = llms.internal.jsonSchemaFromPrototype(nvp.ResponseFormat);
+elseif startsWith(string(nvp.ResponseFormat), asManyOfPattern(whitespacePattern)+"{")
+    parameters.format = llms.internal.verbatimJSON(nvp.ResponseFormat);
+end
+
 if ~isempty(nvp.Seed)
     options.seed = nvp.Seed;
 end

diff --git a/+llms/+internal/hasTools.m b/+llms/+internal/hasTools.m
@@ -12,4 +12,31 @@
         Tools
         FunctionsStruct
     end
+
+    methods(Hidden)
+        function mustBeValidFunctionCall(this, functionCall)
+            if ~isempty(functionCall)
+                mustBeTextScalar(functionCall);
+                if isempty(this.FunctionNames)
+                    error("llms:mustSetFunctionsForCall", llms.utils.errorMessageCatalog.getMessage("llms:mustSetFunctionsForCall"));
+                end
+                mustBeMember(functionCall, ["none","auto", this.FunctionNames]);
+            end
+        end
+
+        function toolChoice = convertToolChoice(this, toolChoice)
+            % if toolChoice is empty
+            if isempty(toolChoice)
+                % if Tools is not empty, the default is 'auto'.
+                if ~isempty(this.Tools)
+                    toolChoice = "auto";
+                end
+            elseif ~ismember(toolChoice,["auto","none"])
+                % if toolChoice is not empty, then it must be "auto", "none" or in the format
+                % {"type": "function", "function": {"name": "my_function"}}
+                toolChoice = struct("type","function","function",struct("name",toolChoice));
+            end
+
+        end
+    end
 end
diff --git a/+llms/+internal/jsonSchemaFromPrototype.m b/+llms/+internal/jsonSchemaFromPrototype.m
@@ -27,7 +27,7 @@
         schema = struct("type","string");
     elseif isinteger(prototype)
         schema = struct("type","integer");
-    elseif isnumeric(prototype)
+    elseif isnumeric(prototype) && ~isa(prototype,'dlarray')
         schema = struct("type","number");
     elseif islogical(prototype)
         schema = struct("type","boolean");

diff --git a/+llms/+internal/sendRequest.m b/+llms/+internal/sendRequest.m
@@ -5,7 +5,7 @@
 %   api key TOKEN. TIMEOUT is the number of seconds to wait for initial
 %   server connection. STREAMFUN is an optional callback function.
 
-%   Copyright 2023-2024 The MathWorks, Inc.
+%   Copyright 2023-2025 The MathWorks, Inc.
 
 arguments
     parameters
@@ -42,4 +42,14 @@
     response = send(request, matlab.net.URI(endpoint),httpOpts,consumer);
     streamedText = consumer.ResponseText;
 end
+
+% When the server sends jsonl or ndjson back, we do not get the automatic conversion.
+if isnumeric(response.Body.Data)
+    txt = native2unicode(response.Body.Data.',"UTF-8");
+    % convert to JSON array
+    json = "[" + replace(strtrim(txt),newline,',') + "]";
+    try
+        response.Body.Data = jsondecode(json);
+    end
+end
 end
diff --git a/+llms/+internal/useSameFieldTypes.m b/+llms/+internal/useSameFieldTypes.m
@@ -21,7 +21,7 @@
     case "struct"
         prototype = prototype(1);
         if isscalar(data)
-            if isequal(fieldnames(data),fieldnames(prototype))
+            if isequal(sort(fieldnames(data)),sort(fieldnames(prototype)))
                 for field_c = fieldnames(data).'
                     field = field_c{1};
                     data.(field) = alignTypes(data.(field),prototype.(field));

diff --git a/+llms/+openai/models.m b/+llms/+openai/models.m
@@ -1,15 +1,19 @@
 function models = models
 %MODELS - supported OpenAI models
 
-%   Copyright 2024 The MathWorks, Inc.
+%   Copyright 2024-2025 The MathWorks, Inc.
     models = [...
-        "gpt-4o","gpt-4o-2024-05-13",...
+        "gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20",...
+        "chatgpt-4o-latest",...
         "gpt-4o-mini","gpt-4o-mini-2024-07-18",...
-        "gpt-4-turbo","gpt-4-turbo-2024-04-09",...
-        "gpt-4","gpt-4-0613", ...
+        "gpt-4-turbo","gpt-4-turbo-2024-04-09","gpt-4-turbo-preview",...
+        "gpt-4","gpt-4-0125-preview","gpt-4-0613","gpt-4-1106-preview",...
         "gpt-3.5-turbo","gpt-3.5-turbo-0125", ...
         "gpt-3.5-turbo-1106",...
-        "o1-preview",...
-        "o1-mini",...
+        "gpt-3.5-turbo-16k",...
+        "o1-preview","o1-preview-2024-09-12",...
+        "o1-mini","o1-mini-2024-09-12",...
+        "o1","o1-2024-12-17",...
+        "o3-mini", "o3-mini-2025-01-31",...
     ];
 end
diff --git a/+llms/+openai/validateResponseFormat.m b/+llms/+openai/validateResponseFormat.m
@@ -9,7 +9,7 @@ function validateResponseFormat(format,model,messages)
     end
 
     if isequal(format, "json")
-        if ismember(model,["gpt-4","gpt-4-0613","o1-preview","o1-mini"])
+        if ismember(model,["gpt-4","gpt-4-0613","o1-preview","o1-mini","o1"])
             error("llms:invalidOptionAndValueForModel", ...
                 llms.utils.errorMessageCatalog.getMessage("llms:invalidOptionAndValueForModel", "ResponseFormat", "json", model));
         elseif nargin > 2
@@ -18,9 +18,6 @@ function validateResponseFormat(format,model,messages)
                 error("llms:warningJsonInstruction", ...
                     llms.utils.errorMessageCatalog.getMessage("llms:warningJsonInstruction"))
             end
-        else
-            warning("llms:warningJsonInstruction", ...
-                llms.utils.errorMessageCatalog.getMessage("llms:warningJsonInstruction"))
         end
     elseif requestsStructuredOutput(format)
         if ~startsWith(model,"gpt-4o")

diff --git a/+llms/+stream/responseStreamer.m b/+llms/+stream/responseStreamer.m
@@ -2,7 +2,7 @@
 %responseStreamer Responsible for obtaining the streaming results from the
 %API
 
-%   Copyright 2023 The MathWorks, Inc.
+%   Copyright 2023-2025 The MathWorks, Inc.
 
     properties
         ResponseText
@@ -97,6 +97,12 @@
                             this.StreamFun(txt);
                             this.ResponseText = [this.ResponseText txt];
                         end
+                        if isfield(json.message,"tool_calls")
+                            s = json.message.tool_calls;
+                            txt = jsonencode(s);
+                            this.StreamFun('');
+                            this.ResponseText = [this.ResponseText txt];
+                        end
                         if isfield(json,"done")
                             stop = json.done;
                         end

diff --git a/+llms/+utils/errorMessageCatalog.m b/+llms/+utils/errorMessageCatalog.m
@@ -1,7 +1,7 @@
 classdef errorMessageCatalog
 %errorMessageCatalog Stores the error messages from this repository
 
-%   Copyright 2023-2024 The MathWorks, Inc.
+%   Copyright 2023-2025 The MathWorks, Inc.
 
     properties(Constant)
         %CATALOG dictionary mapping error ids to error msgs
@@ -28,6 +28,15 @@
                 end
             end
         end
+
+        function s = createCatalog()
+            %createCatalog will run the initialization code and return the catalog
+            %   This is only meant to get more correct test coverage reports:
+            %   The test coverage reports do not include the properties initialization
+            %   for Catalog from above, so we have a test seam here to re-run it
+            %   within the framework, where it is reported.
+            s = buildErrorMessageCatalog;
+        end
     end
 end
 
@@ -40,7 +49,8 @@
 catalog("llms:mustBeAssistantWithContent") = "Input struct must contain field 'content' containing text with one or more characters.";
 catalog("llms:mustBeAssistantWithIdAndFunction") = "Field 'tool_call' must be a struct with fields 'id' and 'function'.";
 catalog("llms:mustBeAssistantWithNameAndArguments") = "Field 'function' must be a struct with fields 'name' and 'arguments'.";
-catalog("llms:assistantMustHaveTextNameAndArguments") = "Fields 'name' and 'arguments' must be text with one or more characters.";
+catalog("llms:assistantMustHaveTextName") = "Field 'name' must be text with one or more characters.";
+catalog("llms:assistantMustHaveTextOrStructArguments") = "Field 'arguments' must be text with one or more characters, or a scalar struct.";
 catalog("llms:mustBeValidIndex") = "Index exceeds the number of array elements. Index must be less than or equal to {1}.";
 catalog("llms:removeFromEmptyHistory") = "Unable to remove message from empty message history.";
 catalog("llms:stopSequencesMustHaveMax4Elements") = "Number of stop sequences must be less than or equal to 4.";
@@ -66,4 +76,5 @@
 catalog("llms:stream:responseStreamer:InvalidInput") = "Input does not have the expected json format, got ""{1}"".";
 catalog("llms:unsupportedDatatypeInPrototype") = "Invalid data type ''{1}'' in prototype. Prototype must be a struct, composed of numerical, string, logical, categorical, or struct.";
 catalog("llms:incorrectResponseFormat") = "Invalid response format. Response format must be ""text"", ""json"", a struct, or a string with a JSON Schema definition.";
+catalog("llms:OllamaStructuredOutputNeeds05") = "Structured output is not supported for Ollama version {1}. Use version 0.5.0 or newer.";
 end
diff --git a/+llms/+utils/requestsStructuredOutput.m b/+llms/+utils/requestsStructuredOutput.m
@@ -0,0 +1,9 @@
+function tf = requestsStructuredOutput(format)
+% This function is undocumented and will change in a future release
+
+% Simple function to check if requested format triggers structured output
+
+%   Copyright 2024 The MathWorks, Inc.
+tf =  isstruct(format) || startsWith(format,asManyOfPattern(whitespacePattern)+"{");
+end
+
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,4 +1,4 @@
 # Code owners, to get auto-filled reviewer lists
 
 # To start with, we just assume everyone in the core team is included on all reviews
-* @adulai @ccreutzi @debymf @MiriamScharnke @vpapanasta
+* @adulai @ccreutzi @debymf @MiriamScharnke @vpapanasta @emanuzzi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,7 +29,7 @@ jobs:
 
       - name: Pull models
         run: |
-            ollama pull mistral
+            ollama pull mistral-nemo
             ollama pull moondream
             OLLAMA_HOST=127.0.0.1:11435 ollama pull qwen2:0.5b
       - name: Set up MATLAB
@@ -39,19 +39,11 @@ jobs:
           cache: true
       - name: Run tests and generate artifacts
         env:
-            OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
             OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }}
             AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_DEPLOYMENT }}
             AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_ENDPOINT }}
             AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_KEY }}
             SECOND_OLLAMA_ENDPOINT: 127.0.0.1:11435
         uses: matlab-actions/run-tests@v2
         with:
-          test-results-junit: test-results/results.xml
-          code-coverage-cobertura: code-coverage/coverage.xml
-          source-folder: .
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: matlab-deep-learning/llms-with-matlab
+          source-folder: .