Skip to content

Commit

Permalink
test(ci): dynamic iteration control for emulator tests
Browse files Browse the repository at this point in the history
this is the same idea from the unit test workflow that lets us
dynamically expand our matrix based on manual workflow input
from GitHub web UI
  • Loading branch information
mikehardy committed Jan 27, 2025
1 parent 687c698 commit 51e1cbd
Showing 1 changed file with 71 additions and 11 deletions.
82 changes: 71 additions & 11 deletions .github/workflows/tests_emulator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@ name: 🛠️ Emulator Tests

on:
workflow_dispatch:
inputs:
iterations:
description: "Number of iterations to run. Default 1. Max 256."
required: true
default: 1
type: number
pull_request:
push:
# Ignore merge queue branches on push; avoids merge_group+push concurrency race since ref is same
Expand All @@ -14,8 +20,71 @@ concurrency:
cancel-in-progress: true

jobs:
# We want to generate our matrix dynamically
# Initial job generates the matrix as a JSON, and following job will use deserialize and use the result
matrix_prep:
# Do not run the scheduled jobs on forks
if: (github.event_name == 'schedule' && github.repository == 'ankidroid/Anki-Android') || (github.event_name != 'schedule')
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.build-matrix.outputs.result }}
steps:
- id: build-matrix
uses: actions/github-script@v7
with:
script: |
// by default, we will run one iteration
let iterationArray = [1]
// workflow dispatch will be a drop-down of different options
if (context.eventName === "workflow_dispatch") {
const inputs = ${{ toJSON(inputs) }}
console.log('inputs is: ' + JSON.stringify(inputs))
const iterationInput = inputs.iterations
console.log('iterations input is: ' + iterationInput)
// this will expand for example with input 5 => [1, 2, 3, 4, 5]
iterationArray = []
for (let i = 1; i <= iterationInput; i++) {
iterationArray.push(i);
}
console.log('iterationArray is: ' + iterationArray)
}
// If we are running on a schedule it's our periodic passive scan for flakes
// Goal is to run enough iterations on all systems that we have confidence there are no flakes
if (context.eventName === "schedule") {
const iterationCount = 15
for (let i = 1; i <= iterationCount; i++) {
iterationArray.push(i);
}
}
// TODO Refactor to make these dynamic with a low/high bracket only on schedule, not push
// For now this is the latest supported API. Previously API 29 was fastest.
// #13695: This was reverted to API 30, 31 was unstable. This should be fixed
let apiLevel = [30];
// These are used for performance tuning what emulator to use.
// try different architectures, targets, delays etc
let arch = [x86_64];
let target = [google_apis];
let firstBootDelay = [600];
return {
"iteration": iterationArray,
"api-level": apiLevel,
"arch": arch,
"target": target,
"first-boot-delay": firstBootDelay
}
- name: Debug Output
run: echo "${{ steps.build-matrix.outputs.result }}"

# This uses the matrix generated from the matrix-prep stage
# it will run unit tests on whatever OS combinations are desired
emulator_test:
name: Android Emulator Test
name: Android Emulator Test (run ${{ matrix.iteration }})
runs-on: ubuntu-latest
timeout-minutes: 75
env:
Expand All @@ -29,16 +98,7 @@ jobs:
KEYALIAS: nrkeystorealias
strategy:
fail-fast: false
matrix:
# Refactor to make these dynamic with a low/high bracket only on schedule, not push
# For now this is the latest supported API. Previously API 29 was fastest.
# #13695: This was reverted to API 30, 31 was unstable. This should be fixed
api-level: [30]
arch: [x86_64]
target: [google_apis]
first-boot-delay: [600]
# This is useful for benchmarking, do 0, 1, 2, etc (up to 256 max job-per-matrix limit) for averages
iteration: [0]
matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
steps:
- name: Free Disk Space (Ubuntu)
uses: insightsengineering/disk-space-reclaimer@v1
Expand Down

0 comments on commit 51e1cbd

Please sign in to comment.