e2e iOS XcFramework

.buildkite/pipeline.yml

@@ -10,12 +10,15 @@ steps:
   - group: ":hammer: Builds"
     steps:
       - label: "XCFramework"
+        key: xcframework
         timeout_in_minutes: 20
         commands:
           - make build_xcframework
         plugins:
           - artifacts#v1.9.3:
-              upload: "BugsnagPerformance.xcframework.zip"
+              upload:
+                - "BugsnagPerformance.xcframework.zip"
+                - "BugsnagPerformanceSwift.xcframework.zip"
 
       - label: "Carthage"
         commands:
@@ -61,6 +64,22 @@ steps:
         artifact_paths:
           - features/fixtures/ios/output/bb_url_swizzling_premain.txt
 
+      - label: "Fixture XcFramework"
+        key: ios_fixture_xcframework
+        depends_on:
+          - xcframework
+        commands:
+          - ./features/fixtures/iosXcFramework/build.sh
+          - bundle install
+          - bundle exec upload-app --farm=bb --app=./features/fixtures/iosXcFramework/output/Fixture.ipa --app-id-file=./features/fixtures/iosXcFramework/output/bb_xcframework_ipa_url.txt
+        plugins:
+          - artifacts#v1.9.3:
+              download:
+                - "BugsnagPerformance.xcframework.zip"
+                - "BugsnagPerformanceSwift.xcframework.zip"
+        artifact_paths:
+          - features/fixtures/iosXcFramework/output/bb_xcframework_ipa_url.txt
+
   - group: ":xcode_simulator: Unit Tests"
     steps:
       - label: "iOS 17 Unit Tests"
@@ -351,6 +370,123 @@ steps:
         concurrency_group: bitbar-app
         concurrency_method: eager
 
+  - group: ":bitbar: XcFramework E2E Tests"
+    steps:
+      - label: ":bitbar: iOS 16 XcFramework E2E Tests"
+        depends_on:
+          - ios_fixture_xcframework
+        timeout_in_minutes: 30
+        agents:
+          queue: opensource
+        plugins:
+          artifacts#v1.9.3:
+            download: "features/fixtures/iosXcFramework/output/bb_xcframework_ipa_url.txt"
+            upload:
+              - "maze_output/failed/**/*"
+              - "maze_output/metrics.csv"
+              - "maze_output/maze_output.zip"
+          docker-compose#v4.8.0:
+            pull: maze-runner-bb-xcframework
+            run: maze-runner-bb-xcframework
+            service-ports: true
+            command:
+              - "--app=@build/bb_xcframework_ipa_url.txt"
+              - "--device=IOS_16"
+              - "--fail-fast"
+              - "--farm=bb"
+              - "--no-tunnel"
+              - "--aws-public-ip"
+              - "features/default"
+        concurrency: 25
+        concurrency_group: bitbar-app
+        concurrency_method: eager
+
+      - label: ":bitbar: iOS 15 XcFramework E2E Tests"
+        depends_on:
+          - ios_fixture_xcframework
+        timeout_in_minutes: 30
+        agents:
+          queue: opensource
+        plugins:
+          artifacts#v1.9.3:
+            download: "features/fixtures/iosXcFramework/output/bb_xcframework_ipa_url.txt"
+            upload:
+              - "maze_output/failed/**/*"
+              - "maze_output/metrics.csv"
+              - "maze_output/maze_output.zip"
+          docker-compose#v4.8.0:
+            pull: maze-runner-bb-xcframework
+            run: maze-runner-bb-xcframework
+            service-ports: true
+            command:
+              - "--app=@build/bb_xcframework_ipa_url.txt"
+              - "--device=IOS_15"
+              - "--fail-fast"
+              - "--farm=bb"
+              - "--no-tunnel"
+              - "--aws-public-ip"
+              - "features/default"
+        concurrency: 25
+        concurrency_group: bitbar-app
+        concurrency_method: eager
+
+      - label: ":bitbar: iOS 14 XcFramework E2E Tests"
+        depends_on:
+          - ios_fixture_xcframework
+        timeout_in_minutes: 30
+        agents:
+          queue: opensource
+        plugins:
+          artifacts#v1.9.3:
+            download: "features/fixtures/iosXcFramework/output/bb_xcframework_ipa_url.txt"
+            upload:
+              - "maze_output/failed/**/*"
+              - "maze_output/metrics.csv"
+              - "maze_output/maze_output.zip"
+          docker-compose#v4.8.0:
+            pull: maze-runner-bb-xcframework
+            run: maze-runner-bb-xcframework
+            service-ports: true
+            command:
+              - "--app=@build/bb_xcframework_ipa_url.txt"
+              - "--device=IOS_14"
+              - "--fail-fast"
+              - "--farm=bb"
+              - "--no-tunnel"
+              - "--aws-public-ip"
+              - "features/default"
+        concurrency: 25
+        concurrency_group: bitbar-app
+        concurrency_method: eager
+
+      - label: ":bitbar: iOS 13 XcFramework E2E Tests"
+        depends_on:
+          - ios_fixture_xcframework
+        timeout_in_minutes: 30
+        agents:
+          queue: opensource
+        plugins:
+          artifacts#v1.9.3:
+            download: "features/fixtures/iosXcFramework/output/bb_xcframework_ipa_url.txt"
+            upload:
+              - "maze_output/failed/**/*"
+              - "maze_output/metrics.csv"
+              - "maze_output/maze_output.zip"
+          docker-compose#v4.8.0:
+            pull: maze-runner-bb-xcframework
+            run: maze-runner-bb-xcframework
+            service-ports: true
+            command:
+              - "--app=@build/bb_xcframework_ipa_url.txt"
+              - "--device=IOS_13"
+              - "--fail-fast"
+              - "--farm=bb"
+              - "--no-tunnel"
+              - "--aws-public-ip"
+              - "features/default"
+        concurrency: 25
+        concurrency_group: bitbar-app
+        concurrency_method: eager
   #
   # BrowserStack
   # TODO - To be moved to BitBar once they provide iOS 17.

docker-compose.yml

@@ -38,6 +38,21 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./reports/:/app/reports/
 
+  maze-runner-bb-xcframework:
+    image: 855461928731.dkr.ecr.us-west-1.amazonaws.com/maze-runner-releases:latest-v9-cli
+    environment:
+      <<: *common-environment
+      BITBAR_USERNAME:
+      BITBAR_ACCESS_KEY:
+    ports:
+      - "9000-9499:9339"
+      - "9000-9499:9340"
+    volumes:
+      - ./features/fixtures/iosXcFramework/output:/app/build
+      - ./features/:/app/features/
+      - ./maze_output:/app/maze_output
+      - /var/run/docker.sock:/var/run/docker.sock
+
   maze-runner-bs:
     image: 855461928731.dkr.ecr.us-west-1.amazonaws.com/maze-runner-releases:latest-v9-cli
     environment:

features/default/automatic_spans.feature

@@ -6,7 +6,8 @@ Feature: Automatic instrumentation spans
     Then the trace "Content-Type" header equals "application/json"
     * the trace "Bugsnag-Span-Sampling" header equals "1:4"
     * the trace "Bugsnag-Sent-At" header matches the regex "^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ$"
-    * a span field "name" equals "[AppStart/iOSCold]"
+#    As we're not able to get a brand new device we're not able to guarantee that its a fresh device
+#    * a span field "name" equals "[AppStart/iOSCold]"
     * a span field "name" equals "[AppStartPhase/App launching - pre main()]"
     * a span field "name" equals "[AppStartPhase/App launching - post main()]"
     * a span field "name" equals "[AppStartPhase/UI init]"
@@ -726,8 +727,8 @@ Feature: Automatic instrumentation spans
     Then the trace "Content-Type" header equals "application/json"
     * the trace "Bugsnag-Span-Sampling" header equals "1:5"
     * the trace "Bugsnag-Sent-At" header matches the regex "^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ$"
-    * a span field "name" equals "[AppStart/iOSCold]"
-    * a span field "name" equals "[AppStartPhase/App launching - pre main()]"
+#    As we're not able to get a brand new device we're not able to guarantee that its a fresh device
+#    * a span field "name" equals "[AppStart/iOSCold]"    * a span field "name" equals "[AppStartPhase/App launching - pre main()]"
     * a span field "name" equals "[AppStartPhase/App launching - post main()]"
     * a span field "name" equals "[AppStartPhase/UI init]"
     * a span field "name" equals "[HTTP/GET]"

features/fixtures/iosXcFramework/.gitignore

@@ -0,0 +1,2 @@
+Fixture.xcarchive
+output

features/fixtures/iosXcFramework/ExportOptions.plist

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>compileBitcode</key>
+	<false/>
+	<key>method</key>
+	<string>development</string>
+	<key>signingStyle</key>
+	<string>automatic</string>
+	<key>stripSwiftSymbols</key>
+	<false/>
+	<key>teamID</key>
+	<string>7W9PZ27Y5F</string>
+	<key>thinning</key>
+	<string>&lt;none&gt;</string>
+</dict>
+</plist>