Add some more complex testing scenarios (#32)

mattleibow · Aug 21, 2024 · b3774fd · b3774fd
1 parent 229620e
commit b3774fd
Show file tree

Hide file tree

Showing 109 changed files with 1,691 additions and 139 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,36 +9,35 @@ on:
 
 jobs:
 
-  windows:
-    name: Windows
-    runs-on: windows-latest
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: 
+          - name: Windows
+            os: windows-latest
+          - name: macOS
+            os: macos-14
+        configuration: 
+          - Debug
+          - Release
+        # Tries to AOT the test infrastructure assemblies which is not supported
+        exclude:
+          - platform:
+              name: macOS
+            configuration: Release
+    name: ${{ matrix.platform.name }} (${{ matrix.configuration }})
+    runs-on: ${{ matrix.platform.os }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - name: Build
-      run: dotnet build DeviceRunners.sln --configuration Release /bl:./artifacts/logs/msbuild-build.binlog
+      run: dotnet build DeviceRunners.sln --configuration ${{ matrix.configuration }} /bl:./artifacts/logs/msbuild-build.binlog
     - name: Upload Logs
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: Logs - Windows
-        path: ./artifacts/logs
-
-  macos:
-    name: macOS
-    runs-on: macos-14
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Setup Required Tools
-      uses: ./.github/workflows/setup-tools
-    - name: Build
-      run: dotnet build DeviceRunners.sln --configuration Debug /bl:./artifacts/logs/msbuild-build.binlog
-    - name: Upload Logs
-      uses: actions/upload-artifact@v1
-      if: ${{ always() }}
-      with:
-        name: Logs - macOS
+        name: Logs - ${{ matrix.platform.name }} (${{ matrix.configuration }})
         path: ./artifacts/logs
diff --git a/.github/workflows/pack.yml b/.github/workflows/pack.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: windows-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - name: Package NuGets
@@ -30,12 +30,12 @@ jobs:
         
         dotnet pack --output ./artifacts/nugets --configuration Release -p:PackageVersion=$VERSION /bl:./artifacts/logs/msbuild-pack.binlog
     - name: Upload NuGet Artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       with:
         name: NuGet
         path: ./artifacts/nugets
     - name: Upload Logs
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
         name: Pack Logs - Windows
@@ -55,7 +55,7 @@ jobs:
         with:
           name: NuGet
       - name: Setup .NET
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: '8.0.x'
           source-url: ${{ env.NUGET_URL }}

diff --git a/.github/workflows/setup-tools/action.yml b/.github/workflows/setup-tools/action.yml
@@ -11,9 +11,15 @@ runs:
     with:
       xcode-version: latest-stable
   - name: Setup .NET
-    uses: actions/setup-dotnet@v3
+    uses: actions/setup-dotnet@v4
     with:
       dotnet-version: 8.0.304
+  - name: Show .NET Info
+    shell: pwsh
+    run: dotnet new globaljson --sdk-version 8.0.304
+  - name: Show .NET Info
+    shell: pwsh
+    run: dotnet --info
   - name: Setup .NET MAUI
     if: runner.os == 'Linux'
     shell: pwsh

diff --git a/.github/workflows/test-xharness-android.yml b/.github/workflows/test-xharness-android.yml
@@ -11,6 +11,7 @@ jobs:
 
   android:
     strategy:
+      fail-fast: false
       matrix:
         platform: 
           - name: Linux
@@ -19,7 +20,7 @@ jobs:
           - name: macOS
             os: macos-13
             gpu: guest
-    name: Android (${{ matrix.platform.name}})
+    name: Android (${{ matrix.platform.name }})
     runs-on: ${{ matrix.platform.os }}
     env:
       TEST_TARGET_FRAMEWORK: net8.0-android
@@ -29,7 +30,7 @@ jobs:
       TEST_EMULATOR_IMAGE: system-images;android-34;google_apis;x86_64
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - uses: actions/setup-java@v4
@@ -50,7 +51,7 @@ jobs:
       run: dotnet android avd start --name ${{ env.TEST_EMULATOR_NAME }} --gpu ${{ matrix.platform.gpu }} --wait-boot --no-window --no-snapshot --no-audio --no-boot-anim
     - name: Publish App
       run: |
-        dotnet publish sample/SampleMauiApp/SampleMauiApp.csproj \
+        dotnet publish sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj \
           -f ${{ env.TEST_TARGET_FRAMEWORK }} \
           -r ${{ env.TEST_RUNTIME_IDENTIFIER }} \
           -c ${{ env.TEST_CONFIGURATION }} \
@@ -59,18 +60,23 @@ jobs:
           /bl:./artifacts/logs/msbuild-publish.binlog
     - name: Run Tests
       run: |
-        dotnet xharness android test \
-          --timeout="00:05:00" \
-          --launch-timeout=00:10:00 \
-          --package-name com.companyname.samplemauiapp \
-          --instrumentation devicerunners.xharness.maui.XHarnessInstrumentation \
-          --app sample/SampleMauiApp/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/publish/com.companyname.samplemauiapp-Signed.apk \
-          --output-directory artifacts
+        for i in $(seq 1 3)
+        do
+          dotnet xharness android test \
+            --timeout="00:05:00" \
+            --launch-timeout=00:10:00 \
+            --package-name com.companyname.devicetestingkitapp.devicetests \
+            --instrumentation devicerunners.xharness.maui.XHarnessInstrumentation \
+            --app sample/test/DeviceTestingKitApp.DeviceTests/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/publish/com.companyname.devicetestingkitapp.devicetests-Signed.apk \
+            --output-directory artifacts \
+          && code=0 && break || code=$? && sleep 15
+        done
+        exit $code
     - name: Shutdown the Emulator
       run: dotnet xharness android adb -- emu kill
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: Test Results - Android (Linux)
+        name: Test Results - ${{ matrix.platform.name }}
         path: ./artifacts
diff --git a/.github/workflows/test-xharness-ios.yml b/.github/workflows/test-xharness-ios.yml
@@ -18,12 +18,12 @@ jobs:
       TEST_CONFIGURATION: Debug
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - name: Publish App
       run: |
-        dotnet build sample/SampleMauiApp/SampleMauiApp.csproj \
+        dotnet build sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj \
           -f ${{ env.TEST_TARGET_FRAMEWORK }} \
           -r ${{ env.TEST_RUNTIME_IDENTIFIER }} \
           -c ${{ env.TEST_CONFIGURATION }} \
@@ -37,14 +37,14 @@ jobs:
             --target ios-simulator-64 \
             --timeout="00:03:00" \
             --launch-timeout=00:06:00 \
-            --app sample/SampleMauiApp/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/SampleMauiApp.app \
+            --app sample/test/DeviceTestingKitApp.DeviceTests/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/DeviceTestingKitApp.DeviceTests.app \
             --output-directory artifacts \
           && code=0 && break || code=$? && sleep 15
         done
         exit $code
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: Test Results - iOS
+        name: Test Results
         path: ./artifacts
diff --git a/.github/workflows/test-xharness-maccatalyst.yml b/.github/workflows/test-xharness-maccatalyst.yml
@@ -18,12 +18,12 @@ jobs:
       TEST_CONFIGURATION: Debug
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - name: Publish App
       run: |
-        dotnet build sample/SampleMauiApp/SampleMauiApp.csproj \
+        dotnet build sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj \
           -f ${{ env.TEST_TARGET_FRAMEWORK }} \
           -r ${{ env.TEST_RUNTIME_IDENTIFIER }} \
           -c ${{ env.TEST_CONFIGURATION }} \
@@ -35,11 +35,11 @@ jobs:
           --target maccatalyst \
           --timeout="00:02:00" \
           --launch-timeout=00:06:00 \
-          --app sample/SampleMauiApp/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/SampleMauiApp.app \
+          --app sample/test/DeviceTestingKitApp.DeviceTests/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/DeviceTestingKitApp.DeviceTests.app \
           --output-directory artifacts
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: Test Results - Mac Catalyst
+        name: Test Results
         path: ./artifacts
diff --git a/.github/workflows/test-xharness-windows.yml b/.github/workflows/test-xharness-windows.yml
@@ -18,13 +18,13 @@ jobs:
       TEST_CONFIGURATION: Release
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - name: Publish App
       run: |
-        $fingerprint = ./scripts/New-Certificate.ps1 -Project sample/SampleMauiApp/SampleMauiApp.csproj
-        dotnet publish sample/SampleMauiApp/SampleMauiApp.csproj `
+        $fingerprint = ./scripts/New-Certificate.ps1 -Project sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj
+        dotnet publish sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj `
           -f ${{ env.TEST_TARGET_FRAMEWORK }} `
           -c ${{ env.TEST_CONFIGURATION }} `
           -p:TestingMode=XHarness `
@@ -35,15 +35,15 @@ jobs:
         ./scripts/Remove-Certificate.ps1 -CertificateFingerprint $fingerprint
     - name: Run Tests
       run: |
-        $msix = Get-ChildItem "sample/SampleMauiApp/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/AppPackages/*_Test/*.msix"
+        $msix = Get-ChildItem "sample/test/DeviceTestingKitApp.DeviceTests/bin/${{ env.TEST_CONFIGURATION }}/${{ env.TEST_TARGET_FRAMEWORK }}/${{ env.TEST_RUNTIME_IDENTIFIER }}/AppPackages/*_Test/*.msix"
         ./scripts/Start-Tests.ps1 `
           -App $msix `
           -OutputDirectory artifacts `
           -TestingMode XHarness
       shell: powershell
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: Test Results - Windows
+        name: Test Results
         path: ./artifacts
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -9,36 +9,27 @@ on:
 
 jobs:
 
-  windows:
-    name: Windows
-    runs-on: windows-latest
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: 
+          - name: Windows
+            os: windows-latest
+          - name: macOS
+            os: macos-14
+    name: ${{ matrix.platform.name }}
+    runs-on: ${{ matrix.platform.os }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Required Tools
       uses: ./.github/workflows/setup-tools
     - name: Test
       run: dotnet test test/DeviceRunners.VisualRunners.Tests --configuration Release /bl:./artifacts/logs/msbuild-test.binlog
     - name: Upload Logs
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: Logs - Windows
-        path: ./artifacts/logs
-
-  macos:
-    name: macOS
-    runs-on: macos-13
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Setup Required Tools
-      uses: ./.github/workflows/setup-tools
-    - name: Test
-      run: dotnet test test/DeviceRunners.VisualRunners.Tests --configuration Release /bl:./artifacts/logs/msbuild-test.binlog
-    - name: Upload Logs
-      uses: actions/upload-artifact@v1
-      if: ${{ always() }}
-      with:
-        name: Logs - macOS
+        name: Logs - ${{ matrix.platform.name }}
         path: ./artifacts/logs