From dd2e01913aa3f09575a51d3e761fe9d83f6871d0 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Tue, 21 May 2024 17:03:54 -0700 Subject: [PATCH] cherry-pick for 0.3.0-rc1 (#495) --- .../stages/jobs/nuget-packaging-job.yml | 7 -- .../stages/jobs/steps/nuget-win-step.yml | 3 +- .../stages/jobs/steps/utils/download-ort.yml | 8 ++ examples/c/src/main.cpp | 6 +- examples/c/src/phi3v.cpp | 3 + examples/csharp/HelloPhi3V/HelloPhi3V.csproj | 14 ++++ examples/csharp/HelloPhi3V/HelloPhi3V.sln | 25 +++++++ examples/csharp/HelloPhi3V/Program.cs | 74 +++++++++++++++++++ nuget/Microsoft.ML.OnnxRuntimeGenAI.nuspec | 4 +- src/csharp/GeneratorParams.cs | 5 ++ src/csharp/Images.cs | 44 +++++++++++ src/csharp/MultiModalProcessor.cs | 73 ++++++++++++++++++ src/csharp/NativeMethods.cs | 44 ++++++++++- src/csharp/Tensor.cs | 30 ++++++++ src/csharp/Utils.cs | 2 + 15 files changed, 326 insertions(+), 16 deletions(-) create mode 100644 examples/csharp/HelloPhi3V/HelloPhi3V.csproj create mode 100644 examples/csharp/HelloPhi3V/HelloPhi3V.sln create mode 100644 examples/csharp/HelloPhi3V/Program.cs create mode 100644 src/csharp/Images.cs create mode 100644 src/csharp/MultiModalProcessor.cs diff --git a/.pipelines/stages/jobs/nuget-packaging-job.yml b/.pipelines/stages/jobs/nuget-packaging-job.yml index 3052d575b..2c81511cf 100644 --- a/.pipelines/stages/jobs/nuget-packaging-job.yml +++ b/.pipelines/stages/jobs/nuget-packaging-job.yml @@ -64,13 +64,6 @@ jobs: value: '.Cuda' ${{ if eq(parameters.ep, 'directml') }}: value: '.DirectML' - - name: ort_nuget_ext - ${{ if eq(parameters.ep, 'cpu') }}: - value: '' - ${{ if eq(parameters.ep, 'cuda') }}: - value: '.Gpu' - ${{ if eq(parameters.ep, 'directml') }}: - value: '.DirectML' - name: ortHome value: 'ort' - name: dml_dir diff --git a/.pipelines/stages/jobs/steps/nuget-win-step.yml b/.pipelines/stages/jobs/steps/nuget-win-step.yml index 86428cbd8..aaff2e17e 100644 --- a/.pipelines/stages/jobs/steps/nuget-win-step.yml +++ b/.pipelines/stages/jobs/steps/nuget-win-step.yml @@ -22,8 +22,7 @@ steps: -Prop genai_nuget_ext=$(genai_nuget_ext) ` -Prop configuration=$(buildConfig) ` -Prop buildPath=$(buildDir) ` - -Prop ort_nuget_ext=$(ort_nuget_ext) ` - -Prop ort_version=$(ort_version) + -Prop ortHome=$(ortHome) nuget.exe pack Microsoft.ML.OnnxRuntimeGenAI.Managed.nuspec ` -Prop version=$VERSION ` -Prop configuration=$(buildConfig) diff --git a/.pipelines/stages/jobs/steps/utils/download-ort.yml b/.pipelines/stages/jobs/steps/utils/download-ort.yml index 4ee46fd67..311a05290 100644 --- a/.pipelines/stages/jobs/steps/utils/download-ort.yml +++ b/.pipelines/stages/jobs/steps/utils/download-ort.yml @@ -74,3 +74,11 @@ steps: RemoveSourceFolder: false displayName: 'Remove .pdb files from lib' continueOnError: true + +- task: DeleteFiles@1 + inputs: + SourceFolder: '$(Build.Repository.LocalPath)/ort/lib' + Contents: 'tools' + RemoveSourceFolder: false + displayName: 'Remove tools folder from lib' + continueOnError: true diff --git a/examples/c/src/main.cpp b/examples/c/src/main.cpp index a99779fe8..056337928 100644 --- a/examples/c/src/main.cpp +++ b/examples/c/src/main.cpp @@ -96,14 +96,16 @@ int main(int argc, char** argv) { OgaHandle handle; std::cout << "-------------" << std::endl; - std::cout << "Hello, Phi-2!" << std::endl; + std::cout << "Hello, Phi-3!" << std::endl; std::cout << "-------------" << std::endl; +#ifdef USE_CXX std::cout << "C++ API" << std::endl; CXX_API(argv[1]); - +#else std::cout << "C API" << std::endl; C_API(argv[1]); +#endif return 0; } \ No newline at end of file diff --git a/examples/c/src/phi3v.cpp b/examples/c/src/phi3v.cpp index d5db579e8..3ba7fc93a 100644 --- a/examples/c/src/phi3v.cpp +++ b/examples/c/src/phi3v.cpp @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #include #include #include diff --git a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj new file mode 100644 index 000000000..c20e9981a --- /dev/null +++ b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj @@ -0,0 +1,14 @@ + + + + Exe + net6.0 + enable + enable + + + + + + + diff --git a/examples/csharp/HelloPhi3V/HelloPhi3V.sln b/examples/csharp/HelloPhi3V/HelloPhi3V.sln new file mode 100644 index 000000000..7e7d279d9 --- /dev/null +++ b/examples/csharp/HelloPhi3V/HelloPhi3V.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.9.34902.65 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HelloPhi3V", "HelloPhi3V.csproj", "{75C05439-20D3-44C3-883A-15E150E9F93E}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {75C05439-20D3-44C3-883A-15E150E9F93E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {75C05439-20D3-44C3-883A-15E150E9F93E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {75C05439-20D3-44C3-883A-15E150E9F93E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {75C05439-20D3-44C3-883A-15E150E9F93E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {58510186-ED6B-46C0-8D3D-DB5300239D3A} + EndGlobalSection +EndGlobal diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs new file mode 100644 index 000000000..5b8d5a21e --- /dev/null +++ b/examples/csharp/HelloPhi3V/Program.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.ML.OnnxRuntimeGenAI; + +class Program +{ + static void Run(string modelPath) + { + using Model model = new Model(modelPath); + using MultiModalProcessor processor = new MultiModalProcessor(model); + using var tokenizerStream = processor.CreateStream(); + + while (true) + { + Console.WriteLine("Image Path (leave empty if no image):"); + string imagePath = Console.ReadLine(); + + Images images = null; + if (imagePath == String.Empty) + { + Console.WriteLine("No image provided"); + } + else + { + if (!File.Exists(imagePath)) + { + throw new Exception("Image file not found: " + imagePath); + } + images = Images.Load(imagePath); + } + + Console.WriteLine("Prompt:"); + string text = Console.ReadLine(); + string prompt = "<|user|>\n"; + if (images != null) + { + prompt += "<|image_1|>\n"; + } + prompt += text + "<|end|>\n<|assistant|>\n"; + + Console.WriteLine("Processing image and prompt..."); + var inputTensors = processor.ProcessImages(prompt, images); + + Console.WriteLine("Generating response..."); + using GeneratorParams generatorParams = new GeneratorParams(model); + generatorParams.SetSearchOption("max_length", 3072); + generatorParams.SetInputs(inputTensors); + + using var generator = new Generator(model, generatorParams); + while (!generator.IsDone()) + { + generator.ComputeLogits(); + generator.GenerateNextToken(); + Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1])); + } + } + + } + + static void Main(string[] args) + { + Console.WriteLine("--------------------"); + Console.WriteLine("Hello, Phi-3-Vision!"); + Console.WriteLine("--------------------"); + + if (args.Length != 1) + { + throw new Exception("Usage: .\\HelloPhi3V "); + } + + Run(args[0]); + } +} \ No newline at end of file diff --git a/nuget/Microsoft.ML.OnnxRuntimeGenAI.nuspec b/nuget/Microsoft.ML.OnnxRuntimeGenAI.nuspec index 705c7b79d..14854208d 100644 --- a/nuget/Microsoft.ML.OnnxRuntimeGenAI.nuspec +++ b/nuget/Microsoft.ML.OnnxRuntimeGenAI.nuspec @@ -17,15 +17,12 @@ - - - @@ -37,6 +34,7 @@ + diff --git a/src/csharp/GeneratorParams.cs b/src/csharp/GeneratorParams.cs index 8ebfb2fcc..ac225e21c 100644 --- a/src/csharp/GeneratorParams.cs +++ b/src/csharp/GeneratorParams.cs @@ -55,6 +55,11 @@ public void SetModelInput(string name, Tensor value) Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetModelInput(_generatorParamsHandle, StringUtils.ToUtf8(name), value.Handle)); } + public void SetInputs(NamedTensors namedTensors) + { + Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetInputs(_generatorParamsHandle, namedTensors.Handle)); + } + ~GeneratorParams() { Dispose(false); diff --git a/src/csharp/Images.cs b/src/csharp/Images.cs new file mode 100644 index 000000000..fcba37aba --- /dev/null +++ b/src/csharp/Images.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntimeGenAI +{ + public class Images : IDisposable + { + private IntPtr _imagesHandle; + private bool _disposed = false; + + private Images(IntPtr imagesHandle) + { + _imagesHandle = imagesHandle; + } + + internal IntPtr Handle { get { return _imagesHandle; } } + + public static Images Load(string imagePath) + { + Result.VerifySuccess(NativeMethods.OgaLoadImage(StringUtils.ToUtf8(imagePath), out IntPtr imagesHandle)); + return new Images(imagesHandle); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + NativeMethods.OgaDestroyImages(_imagesHandle); + _imagesHandle = IntPtr.Zero; + _disposed = true; + } + } +} diff --git a/src/csharp/MultiModalProcessor.cs b/src/csharp/MultiModalProcessor.cs new file mode 100644 index 000000000..9eae0eb20 --- /dev/null +++ b/src/csharp/MultiModalProcessor.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntimeGenAI +{ + public class MultiModalProcessor : IDisposable + { + private IntPtr _processorHandle; + private bool _disposed = false; + + public MultiModalProcessor(Model model) + { + Result.VerifySuccess(NativeMethods.OgaCreateMultiModalProcessor(model.Handle, out _processorHandle)); + } + + internal IntPtr Handle { get { return _processorHandle; } } + + public NamedTensors ProcessImages(string prompt, Images images) + { + IntPtr imagesHandle = images == null ? IntPtr.Zero : images.Handle; + Result.VerifySuccess(NativeMethods.OgaProcessorProcessImages(_processorHandle, StringUtils.ToUtf8(prompt), + imagesHandle, out IntPtr namedTensorsHandle)); + return new NamedTensors(namedTensorsHandle); + } + + public string Decode(ReadOnlySpan sequence) + { + IntPtr outStr = IntPtr.Zero; + unsafe + { + fixed (int* sequencePtr = sequence) + { + Result.VerifySuccess(NativeMethods.OgaProcessorDecode(_processorHandle, sequencePtr, (UIntPtr)sequence.Length, out outStr)); + } + } + try + { + return StringUtils.FromUtf8(outStr); + } + finally + { + NativeMethods.OgaDestroyString(outStr); + } + } + + public TokenizerStream CreateStream() + { + IntPtr tokenizerStreamHandle = IntPtr.Zero; + Result.VerifySuccess(NativeMethods.OgaCreateTokenizerStreamFromProcessor(_processorHandle, out tokenizerStreamHandle)); + return new TokenizerStream(tokenizerStreamHandle); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + NativeMethods.OgaDestroyMultiModalProcessor(_processorHandle); + _processorHandle = IntPtr.Zero; + _disposed = true; + } + } +} diff --git a/src/csharp/NativeMethods.cs b/src/csharp/NativeMethods.cs index 4e54ce608..ff8c0003c 100644 --- a/src/csharp/NativeMethods.cs +++ b/src/csharp/NativeMethods.cs @@ -76,6 +76,10 @@ internal class NativeLib byte[] /* const char* */ name, IntPtr /* const OgaTensor* */ tensor); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaGeneratorParamsSetInputs(IntPtr /* OgaGeneratorParams* */ generatorParams, + IntPtr /* const OgaNamedTensors* */ named_tensors); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] public static extern IntPtr /* OgaResult* */ OgaCreateGenerator(IntPtr /* const OgaModel* */ model, IntPtr /* const OgaGeneratorParams* */ generatorParams, @@ -144,7 +148,7 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq out IntPtr /* OgaTokenizer** */ tokenizer); [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] - public static extern void OgaDestroyTokenizer(IntPtr /* OgaTokenizer* */ model); + public static extern void OgaDestroyTokenizer(IntPtr /* OgaTokenizer* */ tokenizer); [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] public static extern IntPtr /* OgaResult* */ OgaTokenizerEncode(IntPtr /* const OgaTokenizer* */ tokenizer, @@ -167,6 +171,10 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq public static extern IntPtr /* OgaResult* */ OgaCreateTokenizerStream(IntPtr /* const OgaTokenizer* */ tokenizer, out IntPtr /* OgaTokenizerStream** */ tokenizerStream); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaCreateTokenizerStreamFromProcessor(IntPtr /* const OgaMultiModalProcessor* */ processor, + out IntPtr /* OgaTokenizerStream** */ tokenizerStream); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] public static extern void OgaDestroyTokenizerStream(IntPtr /* OgaTokenizerStream* */ tokenizerStream); @@ -178,7 +186,10 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq out IntPtr /* const char** */ outStr); [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] - public static extern IntPtr /* OgaResult* */ OgaCreateTensorFromBuffer(IntPtr /* data* */ data, long[] shape_dims, UIntPtr shape_dims_count, ElementType element_Type, + public static extern IntPtr /* OgaResult* */ OgaCreateTensorFromBuffer(IntPtr /* data* */ data, + long[] shape_dims, + UIntPtr shape_dims_count, + ElementType element_Type, out IntPtr /* OgaTensor** */ tensor); [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] @@ -204,5 +215,34 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] public static extern void OgaShutdown(); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaCreateMultiModalProcessor(IntPtr /* const OgaModel* */ model, + out IntPtr /* OgaMultiModalProcessor** */ processor); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern void OgaDestroyMultiModalProcessor(IntPtr /* OgaMultiModalProcessor* */ processor); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaProcessorProcessImages(IntPtr /* const OgaMultiModalProcessor* */ processor, + byte[] /* const char* */ prompt, + IntPtr /* const Images* */ images, + out IntPtr /* OgaNamedTensors** */ named_tensors); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern unsafe IntPtr /* OgaResult* */ OgaProcessorDecode(IntPtr /* const OgaMultiModalProcessor* */ processor, + int* /* const int32_t* */ sequence, + UIntPtr /* size_t */ sequenceLength, + out IntPtr /* const char** */ outStr); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaLoadImage(byte[] /* const char* */ image_path, + out IntPtr /* const OgaImages** */ images); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern void OgaDestroyImages(IntPtr /* OgaImages* */ images); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern void OgaDestroyNamedTensors(IntPtr /* OgaNamedTensors* */ named_tensors); } } diff --git a/src/csharp/Tensor.cs b/src/csharp/Tensor.cs index 643a34ca4..ae20a1b8b 100644 --- a/src/csharp/Tensor.cs +++ b/src/csharp/Tensor.cs @@ -70,4 +70,34 @@ protected virtual void Dispose(bool disposing) _disposed = true; } } + + public class NamedTensors : IDisposable + { + private IntPtr _namedTensorsHandle; + private bool _disposed = false; + + internal NamedTensors(IntPtr namedTensorsHandle) + { + _namedTensorsHandle = namedTensorsHandle; + } + + internal IntPtr Handle { get { return _namedTensorsHandle; } } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + NativeMethods.OgaDestroyNamedTensors(_namedTensorsHandle); + _namedTensorsHandle = IntPtr.Zero; + _disposed = true; + } + } } diff --git a/src/csharp/Utils.cs b/src/csharp/Utils.cs index 2a8723280..b84f1d407 100644 --- a/src/csharp/Utils.cs +++ b/src/csharp/Utils.cs @@ -21,6 +21,7 @@ public static void SetCurrentGpuDeviceId(int device_id) { Result.VerifySuccess(NativeMethods.OgaSetCurrentGpuDeviceId(device_id)); } + public static int GetCurrentGpuDeviceId() { IntPtr device_id = IntPtr.Zero; @@ -32,6 +33,7 @@ public static void SetLogBool(string name, bool value) { Result.VerifySuccess(NativeMethods.OgaSetLogBool(StringUtils.ToUtf8(name), value)); } + public static void SetLogString(string name, string value) { Result.VerifySuccess(NativeMethods.OgaSetLogString(StringUtils.ToUtf8(name), StringUtils.ToUtf8(value)));