cherry-pick for 0.3.0-rc1 (#495)

.pipelines/stages/jobs/nuget-packaging-job.yml

@@ -64,13 +64,6 @@ jobs:
       value: '.Cuda'
     ${{ if eq(parameters.ep, 'directml') }}:
       value: '.DirectML'
-  - name: ort_nuget_ext
-    ${{ if eq(parameters.ep, 'cpu') }}:
-      value: ''
-    ${{ if eq(parameters.ep, 'cuda') }}:
-      value: '.Gpu'
-    ${{ if eq(parameters.ep, 'directml') }}:
-      value: '.DirectML'
   - name: ortHome
     value: 'ort'
   - name: dml_dir

.pipelines/stages/jobs/steps/nuget-win-step.yml

@@ -22,8 +22,7 @@ steps:
       -Prop genai_nuget_ext=$(genai_nuget_ext) `
       -Prop configuration=$(buildConfig) `
       -Prop buildPath=$(buildDir) `
-      -Prop ort_nuget_ext=$(ort_nuget_ext) `
-      -Prop ort_version=$(ort_version)
+      -Prop ortHome=$(ortHome)
     nuget.exe pack Microsoft.ML.OnnxRuntimeGenAI.Managed.nuspec `
       -Prop version=$VERSION `
       -Prop configuration=$(buildConfig)

.pipelines/stages/jobs/steps/utils/download-ort.yml

@@ -74,3 +74,11 @@ steps:
     RemoveSourceFolder: false
   displayName: 'Remove .pdb files from lib'
   continueOnError: true
+
+- task: DeleteFiles@1
+  inputs:
+    SourceFolder: '$(Build.Repository.LocalPath)/ort/lib'
+    Contents: 'tools'
+    RemoveSourceFolder: false
+  displayName: 'Remove tools folder from lib'
+  continueOnError: true

examples/c/src/main.cpp

@@ -96,14 +96,16 @@ int main(int argc, char** argv) {
   OgaHandle handle;
 
   std::cout << "-------------" << std::endl;
-  std::cout << "Hello, Phi-2!" << std::endl;
+  std::cout << "Hello, Phi-3!" << std::endl;
   std::cout << "-------------" << std::endl;
 
+#ifdef USE_CXX
   std::cout << "C++ API" << std::endl;
   CXX_API(argv[1]);
-
+#else
   std::cout << "C API" << std::endl;
   C_API(argv[1]);
+#endif
 
   return 0;
 }

examples/c/src/phi3v.cpp

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
 #include <iostream>
 #include <string>
 #include <fstream>

examples/csharp/HelloPhi3V/HelloPhi3V.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" Version="0.3.0" />
+  </ItemGroup>
+
+</Project>

examples/csharp/HelloPhi3V/HelloPhi3V.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.9.34902.65
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HelloPhi3V", "HelloPhi3V.csproj", "{75C05439-20D3-44C3-883A-15E150E9F93E}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{75C05439-20D3-44C3-883A-15E150E9F93E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{75C05439-20D3-44C3-883A-15E150E9F93E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{75C05439-20D3-44C3-883A-15E150E9F93E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{75C05439-20D3-44C3-883A-15E150E9F93E}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {58510186-ED6B-46C0-8D3D-DB5300239D3A}
+	EndGlobalSection
+EndGlobal

examples/csharp/HelloPhi3V/Program.cs

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.ML.OnnxRuntimeGenAI;
+
+class Program
+{
+    static void Run(string modelPath)
+    {
+        using Model model = new Model(modelPath);
+        using MultiModalProcessor processor = new MultiModalProcessor(model);
+        using var tokenizerStream = processor.CreateStream();
+
+        while (true)
+        {
+            Console.WriteLine("Image Path (leave empty if no image):");
+            string imagePath = Console.ReadLine();
+
+            Images images = null;
+            if (imagePath == String.Empty)
+            {
+                Console.WriteLine("No image provided");
+            }
+            else
+            {
+                if (!File.Exists(imagePath))
+                {
+                    throw new Exception("Image file not found: " +  imagePath);
+                }
+                images = Images.Load(imagePath);
+            }
+
+            Console.WriteLine("Prompt:");
+            string text = Console.ReadLine();
+            string prompt = "<|user|>\n";
+            if (images != null)
+            {
+                prompt += "<|image_1|>\n";
+            }
+            prompt += text + "<|end|>\n<|assistant|>\n";
+
+            Console.WriteLine("Processing image and prompt...");
+            var inputTensors = processor.ProcessImages(prompt, images);
+
+            Console.WriteLine("Generating response...");
+            using GeneratorParams generatorParams = new GeneratorParams(model);
+            generatorParams.SetSearchOption("max_length", 3072);
+            generatorParams.SetInputs(inputTensors);
+
+            using var generator = new Generator(model, generatorParams);
+            while (!generator.IsDone())
+            {
+                generator.ComputeLogits();
+                generator.GenerateNextToken();
+                Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
+            }
+        }
+
+    }
+
+    static void Main(string[] args)
+    {
+        Console.WriteLine("--------------------");
+        Console.WriteLine("Hello, Phi-3-Vision!");
+        Console.WriteLine("--------------------");
+
+        if (args.Length != 1)
+        {
+            throw new Exception("Usage: .\\HelloPhi3V <model_path>");
+        }
+
+        Run(args[0]);
+    }
+}

nuget/Microsoft.ML.OnnxRuntimeGenAI.nuspec

@@ -17,15 +17,12 @@
     <dependencies>
       <group targetFramework="NETSTANDARD" >
         <dependency id="Microsoft.ML.OnnxRuntimeGenAI.Managed" version="$version$" />
-        <dependency id="Microsoft.ML.OnnxRuntime$ort_nuget_ext$" version="$ort_version$" />
       </group>
       <group targetFramework="NETCOREAPP" >
         <dependency id="Microsoft.ML.OnnxRuntimeGenAI.Managed" version="$version$" />
-        <dependency id="Microsoft.ML.OnnxRuntime$ort_nuget_ext$" version="$ort_version$" />
       </group>
       <group targetFramework="NETFRAMEWORK" >
         <dependency id="Microsoft.ML.OnnxRuntimeGenAI.Managed" version="$version$" />
-        <dependency id="Microsoft.ML.OnnxRuntime$ort_nuget_ext$" version="$ort_version$" />
       </group>
     </dependencies>
   </metadata>
@@ -37,6 +34,7 @@
     <!-- Windows x64 -->
     <file src="..\$buildPath$\$configuration$\onnxruntime-genai.lib" target="runtimes\win-x64\native" />
     <file src="..\$buildPath$\$configuration$\onnxruntime-genai.dll" target="runtimes\win-x64\native" />
+    <file src="..\$ortHome$\lib\**" target="runtimes\win-x64\native" />
 
 <!--      --><!-- Windows arm64 -->
 <!--     <file src="..\$buildPath$\$configuration$\onnxruntime-genai.lib" target="runtimes\win-arm64\native" /> -->

src/csharp/GeneratorParams.cs

@@ -55,6 +55,11 @@ public void SetModelInput(string name, Tensor value)
             Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetModelInput(_generatorParamsHandle, StringUtils.ToUtf8(name), value.Handle));
         }
 
+        public void SetInputs(NamedTensors namedTensors)
+        {
+            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetInputs(_generatorParamsHandle, namedTensors.Handle));
+        }
+
         ~GeneratorParams()
         {
             Dispose(false);

src/csharp/Images.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntimeGenAI
+{
+    public class Images : IDisposable
+    {
+        private IntPtr _imagesHandle;
+        private bool _disposed = false;
+
+        private Images(IntPtr imagesHandle)
+        {
+            _imagesHandle = imagesHandle;
+        }
+
+        internal IntPtr Handle { get { return _imagesHandle; } }
+
+        public static Images Load(string imagePath)
+        {
+            Result.VerifySuccess(NativeMethods.OgaLoadImage(StringUtils.ToUtf8(imagePath), out IntPtr imagesHandle));
+            return new Images(imagesHandle);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_disposed)
+            {
+                return;
+            }
+            NativeMethods.OgaDestroyImages(_imagesHandle);
+            _imagesHandle = IntPtr.Zero;
+            _disposed = true;
+        }
+    }
+}

src/csharp/MultiModalProcessor.cs

@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntimeGenAI
+{
+    public class MultiModalProcessor : IDisposable
+    {
+        private IntPtr _processorHandle;
+        private bool _disposed = false;
+
+        public MultiModalProcessor(Model model)
+        {
+            Result.VerifySuccess(NativeMethods.OgaCreateMultiModalProcessor(model.Handle, out _processorHandle));
+        }
+
+        internal IntPtr Handle { get { return _processorHandle; } }
+
+        public NamedTensors ProcessImages(string prompt, Images images)
+        {
+            IntPtr imagesHandle = images == null ? IntPtr.Zero : images.Handle;
+            Result.VerifySuccess(NativeMethods.OgaProcessorProcessImages(_processorHandle, StringUtils.ToUtf8(prompt),
+                                                                         imagesHandle, out IntPtr namedTensorsHandle));
+            return new NamedTensors(namedTensorsHandle);
+        }
+
+        public string Decode(ReadOnlySpan<int> sequence)
+        {
+            IntPtr outStr = IntPtr.Zero;
+            unsafe
+            {
+                fixed (int* sequencePtr = sequence)
+                {
+                    Result.VerifySuccess(NativeMethods.OgaProcessorDecode(_processorHandle, sequencePtr, (UIntPtr)sequence.Length, out outStr));
+                }
+            }
+            try
+            {
+                return StringUtils.FromUtf8(outStr);
+            }
+            finally
+            {
+                NativeMethods.OgaDestroyString(outStr);
+            }
+        }
+
+        public TokenizerStream CreateStream()
+        {
+            IntPtr tokenizerStreamHandle = IntPtr.Zero;
+            Result.VerifySuccess(NativeMethods.OgaCreateTokenizerStreamFromProcessor(_processorHandle, out tokenizerStreamHandle));
+            return new TokenizerStream(tokenizerStreamHandle);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_disposed)
+            {
+                return;
+            }
+            NativeMethods.OgaDestroyMultiModalProcessor(_processorHandle);
+            _processorHandle = IntPtr.Zero;
+            _disposed = true;
+        }
+    }
+}