Skip to content

Commit 82d6f56

Browse files
mockersfalice-i-cecile
mockersf
and
alice-i-cecile
authored
Compare screenshots with main on PRs (#13248)
# Objective - Compare screenshots for a few examples between PRs and main ## Solution - Send screenshots taken to a screenshot comparison service - Not completely sure every thing will work at once, but it shouldn't break anything at least - it needs a secret to work, I'll add it if enough people agree with this PR - this PR doesn't change anything on the screenshot selection (load_gltf and breakout currently), this will need rendering folks input and can happen later --------- Co-authored-by: Alice Cecile <[email protected]>
1 parent 89e19aa commit 82d6f56

File tree

4 files changed

+184
-47
lines changed

4 files changed

+184
-47
lines changed

‎.github/workflows/ci-examples.yml

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: CI - examples
2+
3+
on:
4+
merge_group:
5+
pull_request_target:
6+
push:
7+
branches:
8+
- main
9+
- release-*
10+
11+
# Unlike jobs in ci.yml, jobs in this workflow can access secrets. Their definitions are taken from the base branch of a PR, not from the PR itself.
12+
13+
env:
14+
CARGO_TERM_COLOR: always
15+
# If nightly is breaking CI, modify this variable to target a specific nightly version.
16+
NIGHTLY_TOOLCHAIN: nightly
17+
18+
concurrency:
19+
group: ${{github.workflow}}-${{github.ref}}-examples
20+
cancel-in-progress: ${{github.event_name == 'pull_request_target'}}
21+
22+
jobs:
23+
run-examples-macos-metal:
24+
# Explicity use macOS 14 to take advantage of M1 chip.
25+
runs-on: macos-14
26+
timeout-minutes: 30
27+
steps:
28+
- uses: actions/checkout@v4
29+
- uses: dtolnay/rust-toolchain@stable
30+
- name: Disable audio
31+
# Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes
32+
run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch
33+
- name: Build bevy
34+
# this uses the same command as when running the example to ensure build is reused
35+
run: |
36+
TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
37+
- name: Run examples
38+
run: |
39+
for example in .github/example-run/*.ron; do
40+
example_name=`basename $example .ron`
41+
echo -n $example_name > last_example_run
42+
echo "running $example_name - "`date`
43+
time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
44+
sleep 10
45+
if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then
46+
mkdir screenshots-$example_name
47+
mv screenshot-*.png screenshots-$example_name/
48+
fi
49+
done
50+
mkdir traces && mv trace*.json traces/
51+
mkdir screenshots && mv screenshots-* screenshots/
52+
- name: save traces
53+
uses: actions/upload-artifact@v4
54+
with:
55+
name: example-traces-macos
56+
path: traces
57+
- name: save screenshots
58+
uses: actions/upload-artifact@v4
59+
with:
60+
name: screenshots-macos
61+
path: screenshots
62+
- uses: actions/upload-artifact@v4
63+
if: ${{ failure() && github.event_name == 'pull_request' }}
64+
with:
65+
name: example-run-macos
66+
path: example-run/
67+
68+
compare-macos-screenshots:
69+
name: Compare macOS screenshots
70+
needs: [run-examples-macos-metal]
71+
uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml
72+
with:
73+
commit: ${{ github.sha }}
74+
branch: ${{ github.ref_name }}
75+
artifact: screenshots-macos
76+
os: macos
77+
secrets: inherit

‎.github/workflows/ci.yml

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -253,52 +253,6 @@ jobs:
253253
echo 'if you use VSCode, you can also install `Typos Spell Checker'
254254
echo 'You can find the extension here: https://marketplace.visualstudio.com/items?itemName=tekumara.typos-vscode'
255255
256-
257-
run-examples-macos-metal:
258-
# Explicity use macOS 14 to take advantage of M1 chip.
259-
runs-on: macos-14
260-
timeout-minutes: 30
261-
steps:
262-
- uses: actions/checkout@v4
263-
- uses: dtolnay/rust-toolchain@stable
264-
- name: Disable audio
265-
# Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes
266-
run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch
267-
- name: Build bevy
268-
# this uses the same command as when running the example to ensure build is reused
269-
run: |
270-
TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
271-
- name: Run examples
272-
run: |
273-
for example in .github/example-run/*.ron; do
274-
example_name=`basename $example .ron`
275-
echo -n $example_name > last_example_run
276-
echo "running $example_name - "`date`
277-
time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
278-
sleep 10
279-
if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then
280-
mkdir screenshots-$example_name
281-
mv screenshot-*.png screenshots-$example_name/
282-
fi
283-
done
284-
mkdir traces && mv trace*.json traces/
285-
mkdir screenshots && mv screenshots-* screenshots/
286-
- name: save traces
287-
uses: actions/upload-artifact@v4
288-
with:
289-
name: example-traces-macos
290-
path: traces
291-
- name: save screenshots
292-
uses: actions/upload-artifact@v4
293-
with:
294-
name: screenshots-macos
295-
path: screenshots
296-
- uses: actions/upload-artifact@v4
297-
if: ${{ failure() && github.event_name == 'pull_request' }}
298-
with:
299-
name: example-run-macos
300-
path: example-run/
301-
302256
check-doc:
303257
runs-on: ubuntu-latest
304258
timeout-minutes: 30

‎.github/workflows/send-screenshots-to-pixeleagle.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Send Screenshots to Pixel Eagle
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
artifact:
7+
required: true
8+
type: string
9+
commit:
10+
required: true
11+
type: string
12+
branch:
13+
required: true
14+
type: string
15+
os:
16+
required: true
17+
type: string
18+
19+
jobs:
20+
send-to-pixel-eagle:
21+
name: Send screenshots to Pixel Eagle
22+
runs-on: ubuntu-latest
23+
steps:
24+
25+
- name: Download artifact
26+
uses: actions/download-artifact@v4
27+
with:
28+
pattern: ${{ inputs.artifact }}
29+
30+
- name: Send to Pixel Eagle
31+
env:
32+
project: B04F67C0-C054-4A6F-92EC-F599FEC2FD1D
33+
run: |
34+
# Create a new run with its associated metadata
35+
metadata='{"os":"${{ inputs.os }}", "commit": "${{ inputs.commit }}", "branch": "${{ inputs.branch }}"}'
36+
run=`curl https://pixel-eagle.vleue.com/$project/runs --json "$metadata" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.id'`
37+
38+
SAVEIFS=$IFS
39+
40+
cd ${{ inputs.artifact }}
41+
42+
# Read the hashes of the screenshot for fast comparison when they are equal
43+
IFS=$'\n'
44+
# Build a json array of screenshots and their hashes
45+
hashes='[';
46+
for screenshot in $(find . -type f -name "*.png");
47+
do
48+
name=${screenshot:14}
49+
echo $name
50+
hash=`shasum -a 256 $screenshot | awk '{print $1}'`
51+
hashes="$hashes [\"$name\",\"$hash\"],"
52+
done
53+
hashes=`echo $hashes | rev | cut -c 2- | rev`
54+
hashes="$hashes]"
55+
56+
IFS=$SAVEIFS
57+
58+
# Upload screenshots with unknown hashes
59+
curl https://pixel-eagle.vleue.com/$project/runs/$run/hashes --json "$hashes" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.[]|[.name] | @tsv' |
60+
while IFS=$'\t' read -r name; do
61+
name=`echo $name | tr -d '"'`
62+
echo "Uploading $name"
63+
curl https://pixel-eagle.vleue.com/$project/runs/$run/screenshots -F "data=@./$name" -F "screenshot=$name" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }}
64+
echo
65+
done
66+
67+
IFS=$SAVEIFS
68+
69+
cd ..
70+
71+
# Trigger comparison with the main branch on the same os
72+
curl https://pixel-eagle.vleue.com/$project/runs/$run/compare/auto --json '{"os":"<equal>", "branch": "main"}' --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} > pixeleagle.json
73+
74+
# Log results
75+
compared_with=`cat pixeleagle.json | jq '.to'`
76+
77+
status=0
78+
missing=`cat pixeleagle.json | jq '.missing | length'`
79+
if [ ! $missing -eq 0 ]; then
80+
echo "There are $missing missing screenshots"
81+
echo "::warning title=$missing missing screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to"
82+
status=1
83+
fi
84+
85+
diff=`cat pixeleagle.json | jq '.diff | length'`
86+
if [ ! $diff -eq 0 ]; then
87+
echo "There are $diff screenshots with a difference"
88+
echo "::warning title=$diff different screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to"
89+
status=1
90+
fi
91+
92+
echo "created run $run: https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to"
93+
94+
exit $status

‎.github/workflows/validation-jobs.yml

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ jobs:
7878
run: cd examples/mobile/android_example && chmod +x gradlew && ./gradlew build
7979

8080
run-examples-linux-vulkan:
81-
if: ${{ github.event_name == 'merge_group' }}
81+
# also run when pushed to main to update reference screenshots
82+
if: ${{ github.event_name != 'pull_request' }}
8283
runs-on: ubuntu-22.04
8384
timeout-minutes: 30
8485
steps:
@@ -135,6 +136,17 @@ jobs:
135136
name: example-run-linux
136137
path: example-run/
137138

139+
compare-linux-screenshots:
140+
name: Compare Linux screenshots
141+
needs: [run-examples-linux-vulkan]
142+
uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml
143+
with:
144+
commit: ${{ github.sha }}
145+
branch: ${{ github.ref_name }}
146+
artifact: screenshots-linux
147+
os: linux
148+
secrets: inherit
149+
138150
run-examples-on-windows-dx12:
139151
if: ${{ github.event_name == 'merge_group' }}
140152
runs-on: windows-latest

0 commit comments

Comments
 (0)