diff --git a/.pipelines/nuget-publishing.yml b/.pipelines/nuget-publishing.yml index 726493826..22ec44677 100644 --- a/.pipelines/nuget-publishing.yml +++ b/.pipelines/nuget-publishing.yml @@ -38,6 +38,11 @@ parameters: type: boolean default: true +- name: enable_apple_framework + displayName: 'Whether Apple framework for iOS & MacCatalyst is built.' + type: boolean + default: false + - name: ort_version displayName: 'OnnxRuntime version' type: string @@ -89,6 +94,7 @@ stages: enable_win_dml: ${{ parameters.enable_win_dml }} enable_win_arm64: ${{ parameters.enable_win_arm64 }} enable_macos_cpu: ${{ parameters.enable_macos_cpu }} + enable_apple_framework: ${{ parameters.enable_apple_framework }} ort_version: ${{ parameters.ort_version }} ort_cuda_version: ${{ parameters.ort_cuda_version }} ort_dml_version: ${{ parameters.ort_dml_version }} @@ -104,6 +110,7 @@ stages: enable_win_dml: ${{ parameters.enable_win_dml }} enable_win_arm64: ${{ parameters.enable_win_arm64 }} enable_macos_cpu: ${{ parameters.enable_macos_cpu }} + enable_apple_framework: ${{ parameters.enable_apple_framework }} ort_version: ${{ parameters.ort_version }} ort_cuda_version: ${{ parameters.ort_cuda_version }} ort_dml_version: ${{ parameters.ort_dml_version }} diff --git a/.pipelines/stages/capi-packaging-stage.yml b/.pipelines/stages/capi-packaging-stage.yml index 2da3971a8..add22efa9 100644 --- a/.pipelines/stages/capi-packaging-stage.yml +++ b/.pipelines/stages/capi-packaging-stage.yml @@ -13,6 +13,8 @@ parameters: type: boolean - name: enable_macos_cpu type: boolean +- name: enable_apple_framework + type: boolean - name: ort_version type: string - name: ort_cuda_version @@ -111,3 +113,12 @@ stages: ort_version: ${{ parameters.ort_version }} os: 'osx' build_config: ${{ parameters.build_config }} + + - ${{ if eq(parameters.enable_apple_framework, true) }}: + - template: jobs/capi-packaging-job.yml + parameters: + os: 'ios' + ep: 'cpu' + arch: 'arm64' + ort_version: ${{ parameters.ort_version }} + build_config: ${{ parameters.build_config }} diff --git a/.pipelines/stages/jobs/capi-packaging-job.yml b/.pipelines/stages/jobs/capi-packaging-job.yml index aa9e9efa4..73a164467 100644 --- a/.pipelines/stages/jobs/capi-packaging-job.yml +++ b/.pipelines/stages/jobs/capi-packaging-job.yml @@ -14,6 +14,7 @@ parameters: - 'linux' - 'win' - 'osx' + - 'ios' - name: build_config type: string default: 'release' @@ -34,7 +35,7 @@ jobs: pool: 'onnxruntime-genai-windows-vs-2022-arm64' ${{ else }}: pool: 'onnxruntime-Win-CPU-2022' - ${{ if eq(parameters.os, 'osx') }}: + ${{ if or(eq(parameters.os, 'osx'), eq(parameters.os, 'ios')) }}: pool: vmImage: 'macOS-latest' @@ -76,6 +77,14 @@ jobs: ${{ else }}: value: 'Microsoft.ML.OnnxRuntime' + - name: ort_native_package_path + ${{ if eq(parameters.os, 'ios') }}: + value: 'ios' + ${{ elseif eq(parameters.os, 'android') }}: + value: 'android' + ${{ else }}: + value: ${{ parameters.os }}-${{ parameters.arch }} + - name: ortHome value: 'ort' - name: dml_dir @@ -143,4 +152,9 @@ jobs: ep: ${{ parameters.ep }} build_config: ${{ parameters.build_config }} + - ${{ if eq(parameters.os, 'ios') }}: + - template: steps/capi-appleframework-step.yml + parameters: + build_config: ${{ parameters.build_config }} + - template: steps/compliant-and-cleanup-step.yml diff --git a/.pipelines/stages/jobs/nuget-packaging-job.yml b/.pipelines/stages/jobs/nuget-packaging-job.yml index 8599ee207..efa71e52b 100644 --- a/.pipelines/stages/jobs/nuget-packaging-job.yml +++ b/.pipelines/stages/jobs/nuget-packaging-job.yml @@ -37,6 +37,11 @@ parameters: type: boolean default: false +- name: enable_apple_framework + displayName: 'Whether Apple framework for iOS & MacCatalyst is built.' + type: boolean + default: false + - name: ort_version type: string @@ -85,6 +90,7 @@ jobs: value: 'Microsoft.ML.OnnxRuntime.Gpu' ${{ if eq(parameters.ep, 'directml') }}: value: 'Microsoft.ML.OnnxRuntime.DirectML' + steps: - ${{ if and(eq(parameters.enable_win_cpu, true), eq(parameters.ep, 'cpu')) }}: - template: steps/utils/flex-download-pipeline-artifact.yml diff --git a/.pipelines/stages/jobs/py-packaging-job.yml b/.pipelines/stages/jobs/py-packaging-job.yml index db39b9afb..4b8828568 100644 --- a/.pipelines/stages/jobs/py-packaging-job.yml +++ b/.pipelines/stages/jobs/py-packaging-job.yml @@ -116,6 +116,14 @@ jobs: ${{ else }}: value: 'Microsoft.ML.OnnxRuntime' + - name: ort_native_package_path + ${{ if eq(parameters.os, 'ios') }}: + value: 'ios' + ${{ if eq(parameters.os, 'android') }}: + value: 'android' + ${{ else }}: + value: ${{ parameters.os }}-${{ parameters.arch }} + - name: dml_dir value: 'Microsoft.AI.DirectML.1.15.1' - name: dml_zip diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml new file mode 100644 index 000000000..1f2f9fb10 --- /dev/null +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -0,0 +1,42 @@ +parameters: +- name: build_config + type: string + default: 'release' + +steps: + +- checkout: self + clean: true + path: onnxruntime-genai + submodules: recursive + +- template: utils/set-genai-version.yml + +- template: utils/set-nightly-build-option-variable.yml + +- template: utils/download-ort.yml + parameters: + archiveType: 'zip' + +- script: | + set -e + python3 tools/ci_build/github/apple/build_apple_framework.py \ + --build_dir "$(Build.BinariesDirectory)/apple_framework" \ + tools/ci_build/github/apple/default_full_ios_framework_build_settings.json + + mkdir $(Build.BinariesDirectory)/artifacts + mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-genai-ios-xcframework-$(genai_version) + cp -R $(Build.BinariesDirectory)/apple_framework/framework_out/onnxruntime-genai.xcframework \ + $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-genai-ios-xcframework-$(genai_version) + pushd $(Build.BinariesDirectory)/artifacts_staging + zip -vr $(Build.BinariesDirectory)/artifacts/onnxruntime_genai_ios_xcframework.zip \ + onnxruntime-genai-ios-xcframework-$(genai_version) + popd + displayName: 'Build Apple XCFramework' + workingDirectory: '$(Build.Repository.LocalPath)' + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: ONNXRuntime XCFramework' + inputs: + ArtifactName: capi-onnxruntime-genai-ios-xcframework + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' diff --git a/.pipelines/stages/jobs/steps/utils/download-ort.yml b/.pipelines/stages/jobs/steps/utils/download-ort.yml index 47717112c..d9f3c27b4 100644 --- a/.pipelines/stages/jobs/steps/utils/download-ort.yml +++ b/.pipelines/stages/jobs/steps/utils/download-ort.yml @@ -25,7 +25,7 @@ steps: displayName: Unzip OnnxRuntime - task: CopyFiles@2 inputs: - SourceFolder: '$(Build.Repository.LocalPath)/ort/runtimes/$(os)-$(arch)/native' + SourceFolder: '$(Build.Repository.LocalPath)/ort/runtimes/$(ort_native_package_path)/native' TargetFolder: '$(Build.Repository.LocalPath)/ort/lib' - ${{ else }}: - script: | @@ -40,7 +40,7 @@ steps: displayName: Unzip OnnxRuntime - task: CopyFiles@2 inputs: - SourceFolder: '$(Build.Repository.LocalPath)/ort/runtimes/$(os)-$(arch)/native' + SourceFolder: '$(Build.Repository.LocalPath)/ort/runtimes/$(ort_native_package_path)/native' TargetFolder: '$(Build.Repository.LocalPath)/ort/lib' # TODO: Find out why do we need to to have libonnxruntime.so.ort_stable_version - script: | diff --git a/.pipelines/stages/nuget-packaging-stage.yml b/.pipelines/stages/nuget-packaging-stage.yml index 00ad62dd2..9122b2459 100644 --- a/.pipelines/stages/nuget-packaging-stage.yml +++ b/.pipelines/stages/nuget-packaging-stage.yml @@ -34,6 +34,11 @@ parameters: type: boolean default: true +- name: enable_apple_framework + displayName: 'Whether Apple framework for iOS & MacCatalyst is built.' + type: boolean + default: false + - name: ort_version type: string - name: ort_cuda_version @@ -57,6 +62,7 @@ stages: enable_win_cpu: ${{ parameters.enable_win_cpu }} enable_win_arm64: ${{ parameters.enable_win_arm64 }} enable_macos_cpu: ${{ parameters.enable_macos_cpu }} + enable_apple_framework: ${{ parameters.enable_apple_framework }} - ${{ if or(eq(parameters.enable_linux_cuda, true), eq(parameters.enable_win_cuda, true)) }}: - template: jobs/nuget-packaging-job.yml parameters: diff --git a/CMakeLists.txt b/CMakeLists.txt index a3d1bac27..4aba803b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -cmake_minimum_required(VERSION 3.26) +cmake_minimum_required(VERSION 3.28) include(FetchContent) include(CMakeDependentOption) project(Generators LANGUAGES C CXX) @@ -13,6 +13,12 @@ if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_ message(FATAL_ERROR "GCC version must be greater than or equal to 11") endif() +# Avoid warning of Calling FetchContent_Populate(Lib) is deprecated temporarily +# TODO: find a better way to handle the header-only 3rd party deps +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.30.0") + cmake_policy(SET CMP0169 OLD) +endif() + if(MSVC) # DLL initialization errors due to old conda msvcp140.dll dll are a result of the new MSVC compiler # See https://developercommunity.visualstudio.com/t/Access-violation-with-std::mutex::lock-a/10664660#T-N10668856 @@ -46,6 +52,15 @@ if(MSVC) ) endif() +# Suggested by https://gitlab.kitware.com/cmake/cmake/-/issues/20132 +# MacCatalyst is not well supported in CMake +# The error that can emerge without this flag can look like: +# "clang : error : overriding '-mmacosx-version-min=11.0' option with '-target x86_64-apple-ios14.0-macabi' [-Werror,-Woverriding-t-option]" +if (PLATFORM_NAME STREQUAL "macabi") + add_compile_options(-Wno-overriding-t-option) + add_link_options(-Wno-overriding-t-option) +endif() + if(ENABLE_TESTS) # call enable_testing so we can add tests from subdirectories (e.g. test and src/java) # it applies recursively to all subdirectories @@ -91,6 +106,10 @@ else() target_link_libraries(onnxruntime-genai PRIVATE ${ONNXRUNTIME_LIB}) endif() +if(APPLE) +target_link_libraries(onnxruntime-genai PRIVATE "-framework Foundation" "-framework CoreML") +endif() + set_target_properties(onnxruntime-genai PROPERTIES FOLDER "Sources") set_target_properties(onnxruntime-genai-static PROPERTIES FOLDER "Sources") source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${generator_srcs}) diff --git a/build.py b/build.py index 6ccca3965..376e52622 100644 --- a/build.py +++ b/build.py @@ -131,7 +131,12 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript # The following options are mutually exclusive (cross compiling options such as android, ios, etc.) platform_group = parser.add_mutually_exclusive_group() platform_group.add_argument("--android", action="store_true", help="Build for Android") - platform_group.add_argument("--ios", action="store_true", help="Build for ios") + platform_group.add_argument("--ios", action="store_true", help="Build for iOS") + platform_group.add_argument( + "--macos", + choices=["MacOSX", "Catalyst"], + help="Specify the target platform for macOS build. Only specify this argument when --build_apple_framework is present.", + ) # Android options parser.add_argument( @@ -157,23 +162,27 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript # iOS build options parser.add_argument( - "--ios_sysroot", + "--apple_sysroot", default="", help="Specify the location name of the macOS platform SDK to be used", ) parser.add_argument( - "--ios_arch", + "--osx_arch", type=str, help="Specify the Target specific architectures for iOS " "This is only supported on MacOS host", ) parser.add_argument( - "--ios_deployment_target", + "--apple_deploy_target", type=str, help="Specify the minimum version of the target platform " "This is only supported on MacOS host", ) + parser.add_argument( + "--build_apple_framework", action="store_true", help="Build a macOS/iOS framework for the ONNXRuntime." + ) + parser.add_argument( "--arm64", action="store_true", @@ -285,14 +294,14 @@ def _validate_ios_args(args: argparse.Namespace): raise ValueError("A Mac host is required to build for iOS") needed_args = [ - args.ios_sysroot, - args.ios_arch, - args.ios_deployment_target, + args.apple_sysroot, + args.osx_arch, + args.apple_deploy_target, ] arg_names = [ - "--ios_sysroot ", - "--ios_arch ", - "--ios_deployment_target ", + "--apple_sysroot ", + "--osx_arch ", + "--apple_deploy_target ", ] have_required_args = all(_ is not None for _ in needed_args) if not have_required_args: @@ -485,9 +494,26 @@ def update(args: argparse.Namespace, env: dict[str, str]): "-DENABLE_TESTS=OFF", ] + if args.ios or args.macos: + platform_name = "macabi" if args.macos == "Catalyst" else args.apple_sysroot + command += [ + "-DENABLE_PYTHON=OFF", + "-DENABLE_TESTS=OFF", + "-DENABLE_MODEL_BENCHMARK=OFF", + f"-DBUILD_APPLE_FRAMEWORK={'ON' if args.build_apple_framework else 'OFF'}", + "-DPLATFORM_NAME=" + platform_name, + ] + + if args.macos: + command += [ + f"-DCMAKE_OSX_SYSROOT={args.apple_sysroot}", + f"-DCMAKE_OSX_ARCHITECTURES={args.osx_arch}", + f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.apple_deploy_target}", + ] + if args.ios: def _get_opencv_toolchain_file(): - if args.ios_sysroot == "iphoneos": + if args.apple_sysroot == "iphoneos": return ( REPO_ROOT / "cmake" / "external" / "opencv" / "platforms" / "iOS" / "cmake" / "Toolchains" / "Toolchain-iPhoneOS_Xcode.cmake" @@ -498,30 +524,38 @@ def _get_opencv_toolchain_file(): "Toolchains" / "Toolchain-iPhoneSimulator_Xcode.cmake" ) - command += [ "-DCMAKE_SYSTEM_NAME=iOS", - f"-DCMAKE_OSX_SYSROOT={args.ios_sysroot}", - f"-DCMAKE_OSX_ARCHITECTURES={args.ios_arch}", - f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.ios_deployment_target}", - "-DENABLE_PYTHON=OFF", + f"-DIOS_ARCH={args.osx_arch}", + f"-DIPHONEOS_DEPLOYMENT_TARGET={args.apple_deploy_target}", # The following arguments are specific to the OpenCV toolchain file - f"-DIOS_ARCH={args.ios_arch}", - f"-DIPHONEOS_DEPLOYMENT_TARGET={args.ios_deployment_target}", f"-DCMAKE_TOOLCHAIN_FILE={_get_opencv_toolchain_file()}", ] + if args.macos == "Catalyst": + if args.cmake_generator == "Xcode": + raise Exception("Xcode CMake generator ('--cmake_generator Xcode') doesn't support Mac Catalyst build.") + + macabi_target = f"{args.osx_arch}-apple-ios{args.apple_deploy_target}-macabi" + command += [ + "-DCMAKE_CXX_COMPILER_TARGET=" + macabi_target, + "-DCMAKE_C_COMPILER_TARGET=" + macabi_target, + "-DCMAKE_CC_COMPILER_TARGET=" + macabi_target, + f"-DCMAKE_CXX_FLAGS=--target={macabi_target}", + f"-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG --target={macabi_target}", + f"-DCMAKE_C_FLAGS=--target={macabi_target}", + f"-DCMAKE_C_FLAGS_RELEASE=-O3 -DNDEBUG --target={macabi_target}", + f"-DCMAKE_CC_FLAGS=--target={macabi_target}", + f"-DCMAKE_CC_FLAGS_RELEASE=-O3 -DNDEBUG --target={macabi_target}", + "-DMAC_CATALYST=1", + ] + if args.arm64: command += ["-A", "ARM64"] elif args.arm64ec: command += ["-A", "ARM64EC"] if args.arm64 or args.arm64ec: - # Build zlib from source. Otherwise zlib from Python might be used. - # And architecture mismatch will happen. - command += ["-D", "BUILD_ZLIB=ON"] - command += ["-DOPENCV_SKIP_SYSTEM_PROCESSOR_DETECTION=ON"] - if args.test: log.warning( "Cannot test on host build machine for cross-compiled " diff --git a/cmake/Info.plist.in b/cmake/Info.plist.in new file mode 100644 index 000000000..f84bd250c --- /dev/null +++ b/cmake/Info.plist.in @@ -0,0 +1,20 @@ + + + + + CFBundleExecutable + ${MACOSX_FRAMEWORK_NAME} + CFBundleName + ${MACOSX_FRAMEWORK_NAME} + CFBundleIdentifier + ${MACOSX_FRAMEWORK_IDENTIFIER} + CFBundleVersion + ${ORT_VERSION} + CFBundleShortVersionString + ${ORT_VERSION} + CFBundleSignature + ???? + CFBundlePackageType + FMWK + + diff --git a/cmake/deps.txt b/cmake/deps.txt index dcfc06f45..07e72b0c8 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -14,4 +14,4 @@ pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6a googletest;https://github.com/google/googletest/archive/530d5c8c84abd2a46f38583ee817743c9b3a42b4.zip;5e3a61db2aa975cfd0f97ba92c818744e7fa7034 microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5 directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v1.613.1.zip;47653509a3371eabb156360f42faf582f314bf2e -onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;711a2cfa699a768721f6bbb83c92e79b27df376f +onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;8d842d85e39aa36985cedf68f5d9e5dfef6f6d05 diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 579076cfc..8dde244c7 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -87,3 +87,9 @@ FetchContent_Declare( ) set(OCOS_BUILD_PRESET ort_genai) onnxruntime_fetchcontent_makeavailable(onnxruntime_extensions) + +list(APPEND EXTERNAL_LIBRARIES + onnxruntime_extensions + ocos_operators + noexcep_operators +) diff --git a/cmake/global_variables.cmake b/cmake/global_variables.cmake index 81f14bd3d..804c21108 100644 --- a/cmake/global_variables.cmake +++ b/cmake/global_variables.cmake @@ -39,9 +39,22 @@ if(WIN32) set(ONNXRUNTIME_PROVIDERS_CUDA_LIB "onnxruntime_providers_cuda.dll") set(ONNXRUNTIME_PROVIDERS_ROCM_LIB "onnxruntime_providers_rocm.dll") elseif(APPLE) - set(ONNXRUNTIME_LIB "libonnxruntime.dylib") - set(ONNXRUNTIME_PROVIDERS_CUDA_LIB "libonnxruntime_providers_cuda.dylib") - set(ONNXRUNTIME_PROVIDERS_ROCM_LIB "libonnxruntime_providers_rocm.dylib") + if(BUILD_APPLE_FRAMEWORK) + add_library(onnxruntime IMPORTED STATIC) + if(PLATFORM_NAME STREQUAL "macabi") + # The xcframework in cmake doesn't seem to support MacCatalyst. + # Without manually setting the target framework, cmake will be confused and looking for wrong libraries. + # The error looks like: 'Unable to find suitable library in: Info.plist for system name "Darwin"' + set_property(TARGET onnxruntime PROPERTY IMPORTED_LOCATION ${ORT_LIB_DIR}/ios-arm64_x86_64-maccatalyst/onnxruntime.framework) + else() + set_property(TARGET onnxruntime PROPERTY IMPORTED_LOCATION ${ORT_LIB_DIR}/onnxruntime.xcframework) + endif() + set(ONNXRUNTIME_LIB onnxruntime) + else() + set(ONNXRUNTIME_LIB "libonnxruntime.dylib") + set(ONNXRUNTIME_PROVIDERS_CUDA_LIB "libonnxruntime_providers_cuda.dylib") + set(ONNXRUNTIME_PROVIDERS_ROCM_LIB "libonnxruntime_providers_rocm.dylib") + endif() else() set(ONNXRUNTIME_LIB "libonnxruntime.so") set(ONNXRUNTIME_PROVIDERS_CUDA_LIB "libonnxruntime_providers_cuda.so") @@ -57,9 +70,16 @@ file(GLOB generator_srcs CONFIGURE_DEPENDS set(ortgenai_embed_libs "") # shared libs that will be embedded inside the onnxruntime-genai package -if(NOT EXISTS "${ORT_LIB_DIR}/${ONNXRUNTIME_LIB}") - message(FATAL_ERROR "Expected the ONNX Runtime library to be found at ${ORT_LIB_DIR}/${ONNXRUNTIME_LIB}. Actual: Not found.") +if (BUILD_APPLE_FRAMEWORK) + if (NOT EXISTS "${ORT_LIB_DIR}/onnxruntime.xcframework") + message(FATAL_ERROR "Expected the ONNX Runtime XCFramework to be found at ${ORT_LIB_DIR}/onnxruntime.xcframework. Actual: Not found.") + endif() +else() + if(NOT EXISTS "${ORT_LIB_DIR}/${ONNXRUNTIME_LIB}") + message(FATAL_ERROR "Expected the ONNX Runtime library to be found at ${ORT_LIB_DIR}/${ONNXRUNTIME_LIB}. Actual: Not found.") + endif() endif() + if(NOT EXISTS "${ORT_HEADER_DIR}/onnxruntime_c_api.h") message(FATAL_ERROR "Expected the ONNX Runtime C API header to be found at \"${ORT_HEADER_DIR}/onnxruntime_c_api.h\". Actual: Not found.") endif() @@ -79,9 +99,9 @@ if (MSVC) elseif (genai_target_platform STREQUAL "ARM64" OR genai_target_platform STREQUAL "ARM64EC") set(genai_target_platform "arm64") - elseif (genai_target_platform STREQUAL "x64" OR - genai_target_platform STREQUAL "x86_64" OR - genai_target_platform STREQUAL "AMD64" OR + elseif (genai_target_platform STREQUAL "x64" OR + genai_target_platform STREQUAL "x86_64" OR + genai_target_platform STREQUAL "AMD64" OR CMAKE_GENERATOR MATCHES "Win64") set(genai_target_platform "x64") else() diff --git a/cmake/package.cmake b/cmake/package.cmake index cb6a0f236..146cb3abc 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -1,6 +1,10 @@ +set(ONNXRUNTIME_GENAI_PUBLIC_HEADERS + "${PROJECT_SOURCE_DIR}/src/ort_genai_c.h;${PROJECT_SOURCE_DIR}/src/ort_genai.h" +) + set_target_properties( onnxruntime-genai PROPERTIES PUBLIC_HEADER - "${PROJECT_SOURCE_DIR}/src/ort_genai_c.h;${PROJECT_SOURCE_DIR}/src/ort_genai.h" + ONNXRUNTIME_GENAI_PUBLIC_HEADERS ) install(TARGETS onnxruntime-genai @@ -8,7 +12,9 @@ install(TARGETS RUNTIME DESTINATION lib ARCHIVE DESTINATION lib PUBLIC_HEADER DESTINATION include + FRAMEWORK DESTINATION ${CMAKE_INSTALL_BINDIR} ) + if (WIN32) install(FILES $ DESTINATION lib CONFIGURATIONS RelWithDebInfo Debug) endif() @@ -69,3 +75,109 @@ install(FILES DESTINATION .) include(CPack) + + +# Assemble the Apple static framework (iOS and macOS) +if(BUILD_APPLE_FRAMEWORK) + # create Info.plist for the framework and podspec for CocoaPods (optional) + set(MACOSX_FRAMEWORK_NAME "onnxruntime-genai") + set(MACOSX_FRAMEWORK_IDENTIFIER "com.microsoft.onnxruntime-genai") + # Need to include CoreML as a weaklink for CocoaPods package if the EP is enabled + if(USE_COREML) + set(APPLE_WEAK_FRAMEWORK "\\\"CoreML\\\"") + endif() + set(INFO_PLIST_PATH "${CMAKE_CURRENT_BINARY_DIR}/Info.plist") + configure_file(${REPO_ROOT}/cmake/Info.plist.in ${INFO_PLIST_PATH}) + configure_file( + ${REPO_ROOT}/tools/ci_build/github/apple/framework_info.json.template + ${CMAKE_CURRENT_BINARY_DIR}/framework_info.json) + + set_target_properties(onnxruntime-genai PROPERTIES + FRAMEWORK TRUE + FRAMEWORK_VERSION A + MACOSX_FRAMEWORK_INFO_PLIST ${INFO_PLIST_PATH} + PUBLIC_HEADER "${ONNXRUNTIME_GENAI_PUBLIC_HEADERS}" + ) + + if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + set_target_properties(onnxruntime-genai PROPERTIES + MACOSX_RPATH TRUE + ) + else() + set_target_properties(onnxruntime-genai PROPERTIES INSTALL_RPATH "@loader_path") + endif() + + # when building for mac catalyst, the CMAKE_OSX_SYSROOT is set to MacOSX as well, to avoid duplication, + # we specify as `-macabi` in the name of the output static apple framework directory. + if (PLATFORM_NAME STREQUAL "macabi") + set(STATIC_FRAMEWORK_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}-macabi) + else() + set(STATIC_FRAMEWORK_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}-${CMAKE_OSX_SYSROOT}) + endif() + + # Setup the various directories required. Remove any existing ones so we start with a clean directory. + set(STATIC_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/static_libraries) + set(STATIC_LIB_TEMP_DIR ${STATIC_LIB_DIR}/temp) + add_custom_command(TARGET onnxruntime-genai PRE_BUILD COMMAND ${CMAKE_COMMAND} -E rm -rf ${STATIC_LIB_DIR}) + add_custom_command(TARGET onnxruntime-genai PRE_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${STATIC_LIB_DIR}) + add_custom_command(TARGET onnxruntime-genai PRE_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${STATIC_LIB_TEMP_DIR}) + + set(STATIC_FRAMEWORK_DIR ${STATIC_FRAMEWORK_OUTPUT_DIR}/static_framework/onnxruntime-genai.framework) + add_custom_command(TARGET onnxruntime-genai PRE_BUILD COMMAND ${CMAKE_COMMAND} -E rm -rf ${STATIC_FRAMEWORK_DIR}) + add_custom_command(TARGET onnxruntime-genai PRE_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${STATIC_FRAMEWORK_DIR}) + + set(INTERNAL_LIBRARIES) + list(APPEND INTERNAL_LIBRARIES onnxruntime-genai-static) + + # If it's an onnxruntime library, extract .o files from the original cmake build path to a separate directory for + # each library to avoid any clashes with filenames (e.g. utils.o) + foreach(_LIB ${INTERNAL_LIBRARIES} ) + GET_TARGET_PROPERTY(_LIB_TYPE ${_LIB} TYPE) + if(_LIB_TYPE STREQUAL "STATIC_LIBRARY") + set(CUR_STATIC_LIB_OBJ_DIR ${STATIC_LIB_TEMP_DIR}/$) + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CUR_STATIC_LIB_OBJ_DIR}) + + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND ar ARGS -x $ + WORKING_DIRECTORY ${CUR_STATIC_LIB_OBJ_DIR}) + endif() + endforeach() + + # for external libraries we create a symlink to the .a file + foreach(_LIB ${EXTERNAL_LIBRARIES}) + GET_TARGET_PROPERTY(_LIB_TYPE ${_LIB} TYPE) + if(_LIB_TYPE STREQUAL "STATIC_LIBRARY") + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + $ ${STATIC_LIB_DIR}/$) + endif() + endforeach() + + # do the pre-link with `ld -r` to create a single relocatable object with correct symbol visibility + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND ld ARGS -r -o ${STATIC_LIB_DIR}/prelinked_objects.o */*.o ../*.a + WORKING_DIRECTORY ${STATIC_LIB_TEMP_DIR}) + + # create the static library + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND libtool -static -o ${STATIC_FRAMEWORK_DIR}/onnxruntime-genai prelinked_objects.o + WORKING_DIRECTORY ${STATIC_LIB_DIR}) + + # Assemble the other pieces of the static framework + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy_if_different ${INFO_PLIST_PATH} ${STATIC_FRAMEWORK_DIR}/Info.plist) + + # add the framework header files + set(STATIC_FRAMEWORK_HEADER_DIR ${STATIC_FRAMEWORK_DIR}/Headers) + file(MAKE_DIRECTORY ${STATIC_FRAMEWORK_HEADER_DIR}) + + foreach(h_ ${ONNXRUNTIME_GENAI_PUBLIC_HEADERS}) + get_filename_component(HEADER_NAME_ ${h_} NAME) + add_custom_command(TARGET onnxruntime-genai POST_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy_if_different ${h_} ${STATIC_FRAMEWORK_HEADER_DIR}/${HEADER_NAME_}) + endforeach() + +endif() \ No newline at end of file diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt index c17a944ad..2b9243c1e 100644 --- a/src/python/CMakeLists.txt +++ b/src/python/CMakeLists.txt @@ -27,12 +27,6 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) target_link_libraries(python PRIVATE cublas curand cudart) endif() -# Avoid warning of Calling FetchContent_Populate(Lib) is deprecated temporarily -# TODO: find a better way to handle the header-only 3rd party deps -if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.30.0") - cmake_policy(SET CMP0169 OLD) -endif() - set_target_properties(python PROPERTIES FOLDER "Sources") source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${python_srcs}) diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py new file mode 100644 index 000000000..ed903dada --- /dev/null +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import glob +import json +import os +import pathlib +import shutil +import subprocess +import sys + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", "..", "..")) +BUILD_PY = os.path.join(REPO_DIR, "build.py") + +# We by default will build below 3 archs +DEFAULT_BUILD_OSX_ARCHS = { + "iphoneos": ["arm64"], + "iphonesimulator": ["arm64", "x86_64"], +} + + +def _parse_build_settings(args): + with open(args.build_settings_file.resolve()) as f: + build_settings_data = json.load(f) + + build_settings = {} + + build_settings["build_osx_archs"] = build_settings_data.get("build_osx_archs", DEFAULT_BUILD_OSX_ARCHS) + + if "build_params" in build_settings_data: + build_settings["build_params"] = build_settings_data["build_params"] + else: + raise ValueError("build_params is required in the build config file") + + return build_settings + + +# Build fat framework for all archs of a single sysroot +# For example, arm64 and x86_64 for iphonesimulator +def _build_for_apple_sysroot( + build_config, intermediates_dir, base_build_command, sysroot, archs, build_dynamic_framework +): + # paths of the onnxruntime libraries for different archs + ort_libs = [] + info_plist_path = "" + + # Build binary for each arch, one by one + for current_arch in archs: + build_dir_current_arch = os.path.join(intermediates_dir, sysroot + "_" + current_arch) + # Use MacOS SDK for Catalyst builds + apple_sysroot = "macosx" if sysroot == "macabi" else sysroot + build_command = [ + *base_build_command, + "--apple_sysroot=" + apple_sysroot, + "--osx_arch=" + current_arch, + "--build_dir=" + build_dir_current_arch, + ] + + # the actual build process for current arch + subprocess.run(build_command, shell=False, check=True, cwd=REPO_DIR) + + # get the compiled lib path + framework_dir = os.path.join( + build_dir_current_arch, + build_config, + build_config + "-" + sysroot, + ( + "onnxruntime-genai.framework" + if build_dynamic_framework + else os.path.join("static_framework", "onnxruntime-genai.framework") + ), + ) + ort_libs.append(os.path.join(framework_dir, "onnxruntime-genai")) + + # We only need to copy Info.plist, framework_info.json, and headers once since they are the same + if not info_plist_path: + info_plist_path = os.path.join(build_dir_current_arch, build_config, "Info.plist") + framework_info_path = os.path.join(build_dir_current_arch, build_config, "framework_info.json") + headers = glob.glob(os.path.join(framework_dir, "Headers", "*.h")) + + # manually create the fat framework + framework_dir = os.path.join(intermediates_dir, "frameworks", sysroot, "onnxruntime-genai.framework") + # remove the existing framework if any + if os.path.exists(framework_dir): + shutil.rmtree(framework_dir) + pathlib.Path(framework_dir).mkdir(parents=True, exist_ok=True) + + # copy the Info.plist, framework_info.json, and header files + + # macos requires different framework structure: + # https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html + if sysroot == "macosx" or sysroot == "macabi": + # create headers and resources directory + header_dir = os.path.join(framework_dir, "Versions", "A", "Headers") + resource_dir = os.path.join(framework_dir, "Versions", "A", "Resources") + pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True) + pathlib.Path(resource_dir).mkdir(parents=True, exist_ok=True) + + shutil.copy(info_plist_path, resource_dir) + shutil.copy(framework_info_path, os.path.dirname(framework_dir)) + + for _header in headers: + shutil.copy(_header, header_dir) + + # use lipo to create a fat ort library + lipo_command = ["lipo", "-create"] + lipo_command += ort_libs + lipo_command += ["-output", os.path.join(framework_dir, "Versions", "A", "onnxruntime-genai")] + subprocess.run(lipo_command, shell=False, check=True) + + # create the symbolic link + pathlib.Path(os.path.join(framework_dir, "Versions", "Current")).symlink_to("A", target_is_directory=True) + pathlib.Path(os.path.join(framework_dir, "Headers")).symlink_to( + "Versions/Current/Headers", target_is_directory=True + ) + pathlib.Path(os.path.join(framework_dir, "Resources")).symlink_to( + "Versions/Current/Resources", target_is_directory=True + ) + pathlib.Path(os.path.join(framework_dir, "onnxruntime-genai")).symlink_to("Versions/Current/onnxruntime-genai") + + else: + shutil.copy(info_plist_path, framework_dir) + shutil.copy(framework_info_path, os.path.dirname(framework_dir)) + header_dir = os.path.join(framework_dir, "Headers") + pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True) + + for _header in headers: + shutil.copy(_header, header_dir) + + # use lipo to create a fat ort library + lipo_command = ["lipo", "-create"] + lipo_command += ort_libs + lipo_command += ["-output", os.path.join(framework_dir, "onnxruntime-genai")] + subprocess.run(lipo_command, shell=False, check=True) + + return framework_dir + + +def _merge_framework_info_files(files, output_file): + merged_data = {} + + for file in files: + with open(file) as f: + data = json.load(f) + for platform, values in data.items(): + assert platform not in merged_data, f"Duplicate platform value: {platform}" + merged_data[platform] = values + + with open(output_file, "w") as f: + json.dump(merged_data, f, indent=2) + + +def _build_package(args): + build_settings = _parse_build_settings(args) + build_dir = os.path.abspath(args.build_dir) + + # Temp dirs to hold building results + intermediates_dir = os.path.join(build_dir, "intermediates") + build_config = args.config + + # build framework for individual sysroot + framework_dirs = [] + framework_info_files_to_merge = [] + public_headers_path = "" + for sysroot in build_settings["build_osx_archs"]: + base_build_command = ( + [sys.executable, BUILD_PY] + + build_settings["build_params"]["base"] + + build_settings["build_params"][sysroot] + + ["--config=" + build_config] + ) + + if args.include_ops_by_config is not None: + base_build_command += ["--include_ops_by_config=" + str(args.include_ops_by_config.resolve())] + + if args.path_to_protoc_exe is not None: + base_build_command += ["--path_to_protoc_exe=" + str(args.path_to_protoc_exe.resolve())] + + framework_dir = _build_for_apple_sysroot( + build_config, + intermediates_dir, + base_build_command, + sysroot, + build_settings["build_osx_archs"][sysroot], + args.build_dynamic_framework, + ) + framework_dirs.append(framework_dir) + + curr_framework_info_path = os.path.join(os.path.dirname(framework_dir), "framework_info.json") + framework_info_files_to_merge.append(curr_framework_info_path) + + # headers for each sysroot are the same, pick one of them + if not public_headers_path: + public_headers_path = os.path.join(os.path.dirname(framework_dir), "onnxruntime-genai.framework", "Headers") + + # create the folder for xcframework and copy the LICENSE and framework_info.json file + xcframework_dir = os.path.join(build_dir, "framework_out") + pathlib.Path(xcframework_dir).mkdir(parents=True, exist_ok=True) + shutil.copy(os.path.join(REPO_DIR, "LICENSE"), xcframework_dir) + shutil.copytree(public_headers_path, os.path.join(xcframework_dir, "Headers"), dirs_exist_ok=True, symlinks=True) + _merge_framework_info_files(framework_info_files_to_merge, os.path.join(build_dir, "xcframework_info.json")) + + # remove existing xcframework if any + xcframework_path = os.path.join(xcframework_dir, "onnxruntime-genai.xcframework") + if os.path.exists(xcframework_path): + shutil.rmtree(xcframework_path) + + # Assemble the final xcframework + build_xcframework_cmd = ["xcrun", "xcodebuild", "-create-xcframework", "-output", xcframework_path] + for framework_dir in framework_dirs: + build_xcframework_cmd.extend(["-framework", framework_dir]) + + subprocess.run(build_xcframework_cmd, shell=False, check=True, cwd=REPO_DIR) + + +def parse_args(): + parser = argparse.ArgumentParser( + os.path.basename(__file__), + description="""Create iOS framework and podspec for one or more osx_archs (xcframework) + and building properties specified in the given build config file, see + tools/ci_build/github/apple/default_full_apple_framework_build_settings.json for details. + The output of the final xcframework and podspec can be found under [build_dir]/framework_out. + Please note, this building script will only work on macOS. + """, + ) + + parser.add_argument( + "--build_dir", + type=pathlib.Path, + default=os.path.join(REPO_DIR, "build/apple_framework"), + help="Provide the root directory for build output", + ) + + parser.add_argument( + "--include_ops_by_config", + type=pathlib.Path, + help="Include ops from config file. See /docs/Reduced_Operator_Kernel_build.md for more information.", + ) + + parser.add_argument( + "--config", + type=str, + default="Release", + choices=["Debug", "MinSizeRel", "Release", "RelWithDebInfo"], + help="Configuration to build.", + ) + + parser.add_argument( + "--build_dynamic_framework", + action="store_true", + help="Build Dynamic Framework (default is build static framework).", + ) + + parser.add_argument( + "build_settings_file", type=pathlib.Path, help="Provide the file contains settings for building iOS framework" + ) + + parser.add_argument("--path_to_protoc_exe", type=pathlib.Path, help="Path to protoc exe.") + + args = parser.parse_args() + + if not args.build_settings_file.resolve().is_file(): + raise FileNotFoundError(f"Build config file {args.build_settings_file.resolve()} is not a file.") + + if args.include_ops_by_config is not None: + include_ops_by_config_file = args.include_ops_by_config.resolve() + if not include_ops_by_config_file.is_file(): + raise FileNotFoundError(f"Include ops config file {include_ops_by_config_file} is not a file.") + + return args + + +def main(): + args = parse_args() + _build_package(args) + + +if __name__ == "__main__": + main() diff --git a/tools/ci_build/github/apple/default_full_apple_framework_build_settings.json b/tools/ci_build/github/apple/default_full_apple_framework_build_settings.json new file mode 100644 index 000000000..b5b1f9dbc --- /dev/null +++ b/tools/ci_build/github/apple/default_full_apple_framework_build_settings.json @@ -0,0 +1,36 @@ +{ + "build_osx_archs": { + "iphoneos": [ + "arm64" + ], + "iphonesimulator": [ + "arm64", + "x86_64" + ], + "macosx": [ + "arm64", + "x86_64" + ] + }, + "build_params": { + "base": [ + "--parallel", + "--cmake_generator", + "Xcode", + "--build_apple_framework", + "--skip_tests" + ], + "macosx": [ + "--macos=MacOSX", + "--apple_deploy_target=11.0" + ], + "iphoneos": [ + "--ios", + "--apple_deploy_target=13.0" + ], + "iphonesimulator": [ + "--ios", + "--apple_deploy_target=13.0" + ] + } +} diff --git a/tools/ci_build/github/apple/default_full_ios_framework_build_settings.json b/tools/ci_build/github/apple/default_full_ios_framework_build_settings.json new file mode 100644 index 000000000..ee9a9f82a --- /dev/null +++ b/tools/ci_build/github/apple/default_full_ios_framework_build_settings.json @@ -0,0 +1,38 @@ +{ + "build_osx_archs": { + "iphoneos": [ + "arm64" + ], + "iphonesimulator": [ + "arm64", + "x86_64" + ], + "macabi": [ + "arm64", + "x86_64" + ] + }, + "build_params": { + "base": [ + "--parallel", + "--build_apple_framework", + "--skip_tests" + ], + "iphoneos": [ + "--ios", + "--cmake_generator", + "Xcode", + "--apple_deploy_target=13.0" + ], + "iphonesimulator": [ + "--ios", + "--cmake_generator", + "Xcode", + "--apple_deploy_target=13.0" + ], + "macabi":[ + "--macos=Catalyst", + "--apple_deploy_target=14.0" + ] + } +} diff --git a/tools/ci_build/github/apple/framework_info.json.template b/tools/ci_build/github/apple/framework_info.json.template new file mode 100644 index 000000000..1f7eeb594 --- /dev/null +++ b/tools/ci_build/github/apple/framework_info.json.template @@ -0,0 +1,6 @@ +{ + "@PLATFORM_NAME@": { + "APPLE_DEPLOYMENT_TARGET": "@CMAKE_OSX_DEPLOYMENT_TARGET@", + "WEAK_FRAMEWORK": "@APPLE_WEAK_FRAMEWORK@" + } +}