From 113aa2992fdf05ff3c871ef722ee1bc2e9462eca Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Tue, 14 May 2024 14:10:56 -0700 Subject: [PATCH] Update React Native CI (#20673) - Move iOS package build to separate job so it can run in parallel with Android AAR build and be decoupled from the test stage. The test stage fails sometimes (not infrequently) and may need to be re-run. - Update stop iOS simulator step so it doesn't fail if the start step doesn't run. --- .../npm-packaging-pipeline.yml | 2 +- .../templates/react-native-ci.yml | 164 ++++++++++-------- 2 files changed, 95 insertions(+), 71 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml index 21fc205c72e89..99d6b416d235c 100644 --- a/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml @@ -56,7 +56,7 @@ stages: BuildConfig: 'Release' PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU' PackageName: 'onnxruntime-react-native' - BuildAndroidAARStageDependsOn: 'Precheck_and_extract_commit' + InitialStageDependsOn: 'Precheck_and_extract_commit' - stage: Download_Node_Package_And_Publish_Validation_Script dependsOn: diff --git a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml index d13cb7a99e7f8..3b00dd741e398 100644 --- a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml @@ -19,16 +19,17 @@ parameters: displayName: 'Package name' type: string default: 'NPM_packages' -- name: BuildAndroidAARStageDependsOn - displayName: 'Stage the Build Android AAR stage depends on' +- name: InitialStageDependsOn + displayName: 'Stage that the initial stage of react-native-ci depends on' type: string default: '' stages: -- stage: Build_Android_AAR - displayName: Build Android AAR - dependsOn: '${{parameters.BuildAndroidAARStageDependsOn}}' +- stage: Build_Packages + displayName: Build Packages + dependsOn: '${{parameters.InitialStageDependsOn}}' jobs: + - template: android-java-api-aar.yml parameters: buildConfig: '${{parameters.BuildConfig}}' @@ -43,10 +44,73 @@ stages: pool_name: '${{parameters.PoolName}}' packageName: 'onnxruntime-android' + - job: Build_Ios_Pod_For_React_Native + pool: + vmImage: 'macOS-12' + + timeoutInMinutes: 90 + + variables: + runCodesignValidationInjection: false + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + ORT_CACHE_DIR: '$(Pipeline.Workspace)/ccache_ort' + + steps: + - template: use-xcode-version.yml + + - task: UsePythonVersion@0 + displayName: Use python 3.9 + inputs: + versionSpec: "3.9" + addToPath: true + architecture: "x64" + + - script: | + pip install -r tools/ci_build/github/apple/ios_packaging.requirements.txt + displayName: "Install Python requirements" + + - template: mac-build-step-with-cache.yml + parameters: + WithCache: true + Today: $(TODAY) + AdditionalKey: react_${{parameters.BuildConfig}} + CacheDir: $(ORT_CACHE_DIR) + BuildStep: + - script: | + set -e -x + python $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_and_assemble_apple_pods.py \ + --build-dir "$(Build.BinariesDirectory)/ios_framework_full" \ + --staging-dir "$(Build.BinariesDirectory)/staging" \ + --variant Full \ + --build-settings-file $(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_full_ios_framework_build_settings.json + + # Mobile build: + # python $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_and_assemble_apple_pods.py \ + # --build_dir $(Build.BinariesDirectory)/ios_framework_mobile \ + # --staging-dir "$(Build.BinariesDirectory)/staging" \ + # --include_ops_by_config $(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config \ + # --variant Mobile \ + # --build-settings-file $(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_mobile_ios_framework_build_settings.json + displayName: Build iOS package and assemble pods + env: + CC: clang + CXX: clang++ + CCACHE_CPP2: 1 + CCACHE_DEPEND: 1 + CCACHE_SLOPPINESS: modules + CCACHE_DIR: $(ORT_CACHE_DIR) + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.BinariesDirectory)/staging' + artifact: 'onnxruntime-ios-full-pod' + # Mobile build: + # artifact: 'onnxruntime-ios-mobile-pod' + - stage: ReactNative_CI displayName: React Native CI dependsOn: - - Build_Android_AAR + - Build_Packages jobs: - job: ReactNative_CI pool: @@ -55,10 +119,7 @@ stages: vmImage: 'macOS-12' variables: runCodesignValidationInjection: false - TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - PROTO_CACHE_DIR: '$(Pipeline.Workspace)/ccache_proto' - ORT_CACHE_DIR: '$(Pipeline.Workspace)/ccache_ort' - timeoutInMinutes: 180 + timeoutInMinutes: 60 steps: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: Clean Agent Directories @@ -98,58 +159,6 @@ stages: brew install applesimutils displayName: Install applesimutils tools required by detox ios - - script: | - python -m pip install -q flatbuffers - displayName: Install python modules - - - template: mac-build-step-with-cache.yml - parameters: - WithCache: true - Today: $(TODAY) - AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh' - CacheDir: $(PROTO_CACHE_DIR) - ChangeEveryCommit: false - BuildStep: - - script: | - set -e -x - $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -d $(Build.SourcesDirectory)/cmake/deps.txt -p $(Build.BinariesDirectory)/installed - displayName: Build protobuf - env: - CCACHE_DIR: $(PROTO_CACHE_DIR) - - - template: mac-build-step-with-cache.yml - parameters: - WithCache: true - Today: $(TODAY) - AdditionalKey: react_${{parameters.BuildConfig}} - CacheDir: $(ORT_CACHE_DIR) - BuildStep: - - script: | - set -e -x - python $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_and_assemble_apple_pods.py \ - --build-dir "$(Build.BinariesDirectory)/ios_framework_full" \ - --staging-dir "$(Build.BinariesDirectory)/staging" \ - --variant Full \ - --build-settings-file $(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_full_ios_framework_build_settings.json \ - -b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/installed/bin/protoc" - - # Mobile build: - # python $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_and_assemble_apple_pods.py \ - # --build_dir $(Build.BinariesDirectory)/ios_framework_mobile \ - # --staging-dir "$(Build.BinariesDirectory)/staging" \ - # --include_ops_by_config $(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config \ - # --variant Mobile \ - # --build-settings-file $(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_mobile_ios_framework_build_settings.json \ - # -b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/installed/bin/protoc" - displayName: Build iOS package and assemble pods - env: - CC: clang - CXX: clang++ - CCACHE_CPP2: 1 - CCACHE_DEPEND: 1 - CCACHE_SLOPPINESS: modules - CCACHE_DIR: $(ORT_CACHE_DIR) - - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' @@ -170,6 +179,16 @@ stages: targetFolder: $(Build.SourcesDirectory)/js/react_native/android/libs displayName: Copy Android package to React Native directory + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + # Mobile build: + # artifactName: 'onnxruntime-ios-mobile-pod' + # targetPath: '$(Build.BinariesDirectory)/ios-mobile-pod' + artifactName: 'onnxruntime-ios-full-pod' + targetPath: '$(Build.BinariesDirectory)/ios-full-pod' + displayName: Download iOS pod artifacts + - script: | npm ci workingDirectory: '$(Build.SourcesDirectory)/js' @@ -206,16 +225,16 @@ stages: - script: | # Mobile build: - # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \ - ORT_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-c \ + # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/ios-mobile-pod/onnxruntime-mobile-c \ + ORT_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/ios-full-pod/onnxruntime-c \ pod install workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios' displayName: Pod install for onnxruntime react native ios bridge library - script: | # Mobile build: - # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \ - ORT_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-c \ + # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/ios-mobile-pod/onnxruntime-mobile-c \ + ORT_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/ios-full-pod/onnxruntime-c \ pod install workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' displayName: Pod install for onnxruntime react native ios e2e tests @@ -299,8 +318,9 @@ stages: # Unit tests and E2E tests with iOS simulator # - script: | - xcrun simctl create iPhoneRNTest com.apple.CoreSimulator.SimDeviceType.iPhone-13 - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' + set -e + DEVICE_ID_FILE="$(Build.BinariesDirectory)/ios_simulator_device_id.txt" + xcrun simctl create iPhoneRNTest com.apple.CoreSimulator.SimDeviceType.iPhone-13 > "${DEVICE_ID_FILE}" displayName: Start iOS Simulator - task: Xcode@5 @@ -334,8 +354,12 @@ stages: displayName: Run React Native Detox iOS e2e Tests - script: | - xcrun simctl delete iPhoneRNTest - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' + set -e + DEVICE_ID_FILE="$(Build.BinariesDirectory)/ios_simulator_device_id.txt" + if [[ -f "${DEVICE_ID_FILE}" ]]; then + xcrun simctl delete "$(cat "${DEVICE_ID_FILE}")" + rm "${DEVICE_ID_FILE}" + fi displayName: Stop iOS Simulator condition: always()