Skip to content

Commit 8b345ed

Browse files
hmdnehmdne
committed
CI: Performance regression (and progression) check
1 parent 6e7cc73 commit 8b345ed

File tree

6 files changed

+160
-31
lines changed

6 files changed

+160
-31
lines changed

‎.github/workflows/build.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,11 @@ jobs:
4949
ruby: 3.0
5050
- name: timezone
5151
ruby: 3.0
52-
- name: optstatus
52+
- name: performance
5353
ruby: 3.0
5454
permissive: true
5555
fetchdepth: '0'
56-
command: bin/rake optstatus_compare
56+
command: bin/rake performance_compare
5757

5858
# Currently failing:
5959
# - ruby: truffleruby

‎Rakefile

+1
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ import 'tasks/building.rake'
77
import 'tasks/linting.rake'
88
import 'tasks/benchmarking.rake'
99
import 'tasks/releasing.rake'
10+
import 'tasks/performance.rake'
1011

1112
task :default => [:rspec, :mspec, :minitest]

‎tasks/performance.rake

+151
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# A set of tasks to track performance regressions by the CI
2+
3+
require 'opal/util'
4+
5+
# Runs a block N times and returns a mean number of seconds it took to run it.
6+
mean_time = proc do |tries: 31, &block|
7+
tries.times.map do
8+
t = Time.now
9+
block.()
10+
Time.now - t
11+
end.sort[tries/2]
12+
end
13+
14+
failed = false
15+
16+
# Mark a failure
17+
failure = proc do |reason|
18+
failed ||= []
19+
failed << reason
20+
end
21+
22+
compare_values = proc do |name, current, previous|
23+
change = ((current - previous).to_f / previous) * 100
24+
25+
puts ("%30s: %.3f -> %.3f (change: %.2f%%)" % [name, previous, current, change]).gsub('.000','')
26+
27+
if change > 5.0
28+
failure.("#{name} increased by more than 5%")
29+
end
30+
end
31+
32+
task :asciidoctor_prepare do
33+
# Selected asciidoctor versions were working on Aug 19 2021, feel free to update.
34+
system("bash", "-c", <<~END)
35+
mkdir -p tmp/performance
36+
pushd tmp/performance
37+
git clone https://github.com/asciidoctor/asciidoctor >/dev/null 2>&1
38+
pushd asciidoctor; git checkout 869e8236 >/dev/null 2>&1; popd
39+
git clone https://github.com/asciidoctor/asciidoctor.js >/dev/null 2>&1
40+
pushd asciidoctor.js; git checkout 053fa0d3 >/dev/null 2>&1; popd
41+
erb ../../tasks/performance/asciidoctor_test.rb.erb > asciidoctor_test.rb
42+
popd
43+
END
44+
end
45+
46+
task :asciidoctor_build_opal do
47+
system("bin/opal -c" \
48+
"-Itmp/performance/asciidoctor/lib " \
49+
"-Itmp/performance/asciidoctor.js/packages/core/lib " \
50+
"tmp/performance/asciidoctor_test.rb > tmp/performance/asciidoctor_test.js")
51+
end
52+
53+
task :asciidoctor_run_ruby do
54+
system("ruby -Itmp/performance/asciidoctor/lib tmp/performance/asciidoctor_test.rb")
55+
end
56+
57+
task :asciidoctor_run_opal do
58+
system("node tmp/performance/asciidoctor_test.js")
59+
end
60+
61+
desc "Generate V8 function optimization status report for corelib methods"
62+
task :optstatus do
63+
system("NODE_OPTS=--allow-natives-syntax bin/opal tasks/performance/optimization_status.rb")
64+
end
65+
66+
task :performance_compare do
67+
this_ref = `git describe --tags`.chomp
68+
ref = 'master'
69+
ref = ENV['GITHUB_BASE_REF'] if ENV['GITHUB_BASE_REF'] && !ENV['GITHUB_BASE_REF'].empty?
70+
71+
# Prepare
72+
system("bundle exec rake asciidoctor_prepare")
73+
system("bundle exec rake asciidoctor_run_ruby > tmp/performance/ruby_result.html")
74+
75+
# Run on current
76+
puts "* Checking optimization status with current..."
77+
system("bundle exec rake optstatus > tmp/performance/optstatus_current")
78+
puts "* Building AsciiDoctor with current..."
79+
compiler_time_current = mean_time.(tries: 7) do
80+
system("bundle exec rake asciidoctor_build_opal")
81+
end
82+
puts "* Running AsciiDoctor with current..."
83+
run_time_current = mean_time.(tries: 31) do
84+
system("bundle exec rake asciidoctor_run_opal > tmp/performance/opal_result_current.html")
85+
end
86+
correct_current = File.read("tmp/performance/opal_result_current.html") == File.read("tmp/performance/ruby_result.html")
87+
size_current = File.size("tmp/performance/asciidoctor_test.js")
88+
puts "* Minifying AsciiDoctor with current..."
89+
min_size_current = Opal::Util.uglify(File.read("tmp/performance/asciidoctor_test.js")).bytesize rescue 133799999999999
90+
91+
# Prepare previous
92+
system("git checkout --recurse-submodules #{ref}")
93+
system("bundle install >/dev/null 2>&1")
94+
95+
# Run on previous
96+
puts "* Checking optimization status with previous..."
97+
system("bundle exec rake optstatus > tmp/performance/optstatus_previous")
98+
puts "* Building AsciiDoctor with previous..."
99+
compiler_time_previous = mean_time.(tries: 7) do
100+
system("bundle exec rake asciidoctor_build_opal")
101+
end
102+
puts "* Running AsciiDoctor with previous..."
103+
run_time_previous = mean_time.(tries: 31) do
104+
system("bundle exec rake asciidoctor_run_opal > tmp/performance/opal_result_previous.html")
105+
end
106+
correct_previous = File.read("tmp/performance/opal_result_previous.html") == File.read("tmp/performance/ruby_result.html")
107+
size_previous = File.size("tmp/performance/asciidoctor_test.js")
108+
puts "* Minifying AsciiDoctor with previous..."
109+
min_size_previous = Opal::Util.uglify(File.read("tmp/performance/asciidoctor_test.js")).bytesize rescue 133799999999999
110+
111+
# Restore current
112+
system("git checkout --recurse-submodules #{this_ref}")
113+
system("bundle install >/dev/null 2>&1")
114+
115+
# Summary
116+
puts
117+
puts "Summary of performance changes between (previous) #{ref} and (current) #{this_ref}:"
118+
119+
diff = `diff -F '^Class' -Naur tmp/performance/optstatus_previous tmp/performance/optstatus_current`
120+
diff_lines = diff.split("\n")
121+
122+
puts
123+
puts "Comparison of V8 function optimization status:"
124+
puts diff
125+
126+
if diff_lines.grep(/^-\s+\[COMPILED\]/).count > 0
127+
failure.("Some methods are no longer compiled on V8")
128+
end
129+
130+
puts
131+
puts "Comparison of the Asciidoctor (a real-life Opal application) compile and run:"
132+
133+
failure.("Wrong result on the current branch") unless correct_current
134+
failure.("Wrong result on the previous branch - ignore it") unless correct_previous
135+
136+
compare_values.("Compile time", compiler_time_current, compiler_time_previous)
137+
compare_values.("Run time", run_time_current, run_time_previous)
138+
compare_values.("Bundle size", size_current, size_previous)
139+
compare_values.("Minified bundle size", min_size_current, min_size_previous)
140+
141+
if failed
142+
puts
143+
puts "This run failed - some performance checks did not pass. Don't worry, this is"
144+
puts "informative, not fatal. It may be worth to rerun the task or consult those"
145+
puts "results with a pull request reviewer:"
146+
failed.each do |f|
147+
puts " - #{f}"
148+
end
149+
fail
150+
end
151+
end

‎tasks/performance/asciidoctor_test.rb.erb

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
JAVASCRIPT_IO_MODULE="nodejs" if RUBY_ENGINE == 'opal'
2+
require "asciidoctor"
3+
puts Asciidoctor.convert(DATA.read)
4+
5+
__END__
6+
<%= File.read('asciidoctor/benchmark/sample-data/mdbasics.adoc') %>

‎tasks/testing/optimization_status.rb ‎tasks/performance/optimization_status.rb

File renamed without changes.

‎tasks/testing.rake

-29
Original file line numberDiff line numberDiff line change
@@ -467,32 +467,3 @@ task :test_all => [:rspec, :mspec, :minitest]
467467
task(:cruby_tests) { warn "The task 'cruby_tests' has been renamed to 'minitest_cruby_nodejs'."; exit 1 }
468468
task(:test_cruby) { warn "The task 'test_cruby' has been renamed to 'minitest_cruby_nodejs'."; exit 1 }
469469
task(:test_nodejs) { warn "The task 'test_nodejs' has been renamed to 'minitest_node_nodejs'."; exit 1 }
470-
471-
desc "Generate V8 function optimization status report for corelib methods"
472-
task :optstatus do
473-
system("NODE_OPTS=--allow-natives-syntax bin/opal tasks/testing/optimization_status.rb")
474-
end
475-
476-
task :optstatus_compare do
477-
this_ref = `git describe`.chomp
478-
ref = "HEAD^"
479-
ref = ENV['GITHUB_BASE_REF'] if ENV['GITHUB_BASE_REF'] && !ENV['GITHUB_BASE_REF'].empty?
480-
ref = `git describe #{ref}`.chomp
481-
482-
system("bundle exec rake optstatus > /tmp/optstatus_current")
483-
system("git checkout --recurse-submodules #{ref}")
484-
system("bundle install >/dev/null 2>&1")
485-
system("bundle exec rake optstatus > /tmp/optstatus_previous")
486-
system("git checkout --recurse-submodules #{this_ref}")
487-
system("bundle install >/dev/null 2>&1")
488-
489-
diff = `diff -F '^Class' -Naur /tmp/optstatus_previous /tmp/optstatus_current`
490-
diff_lines = diff.split("\n")
491-
492-
puts
493-
puts "Comparison of V8 function optimization status between #{ref} and #{this_ref}:"
494-
puts diff
495-
puts
496-
497-
fail if diff_lines.grep(/^-\s+\[COMPILED\]/).count > 0
498-
end

0 commit comments

Comments
 (0)