Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prism benchmarks #30

Merged
merged 3 commits into from
Jul 19, 2024
Merged

Prism benchmarks #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a8bc89
Prism benchmarking script
egiurleo Jul 16, 2024
8ede117
Run benchmarks
egiurleo Jul 18, 2024
38601eb
Remove .gitkeep files
egiurleo Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions prism_benchmarks/data/parser/sorbet-tests/2024-07-18-f75db5824.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
{
"results": [
{
"command": "bazel-bin/main/sorbet --parser=sorbet --stop-after=parser test/testdata",
"mean": 10.0427811339,
"stddev": 0.025126222846626647,
"median": 10.0368188785,
"user": 15.975400439999998,
"system": 1.0591082200000002,
"min": 10.0078213055,
"max": 10.0817527725,
"times": [
10.0276461275,
10.0627562915,
10.0649185675,
10.0174194365,
10.0078213055,
10.0817527725,
10.0454906105,
10.0242698005,
10.0281471465,
10.0675892805
],
"exit_codes": [
100,
100,
100,
100,
100,
100,
100,
100,
100,
100
],
"parameters": {
"parser": "sorbet"
}
},
{
"command": "bazel-bin/main/sorbet --parser=prism --stop-after=parser test/testdata",
"mean": 9.028913599900001,
"stddev": 0.1174086729884074,
"median": 8.991058692,
"user": 12.66185554,
"system": 1.9508827199999998,
"min": 8.9338502675,
"max": 9.2513346015,
"times": [
8.9816991245,
9.041188655500001,
9.0004182595,
9.0065693535,
9.2513346015,
8.9599665045,
8.9354991625,
8.9459703115,
9.2326397585,
8.9338502675
],
"exit_codes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"parameters": {
"parser": "prism"
}
}
]
}
78 changes: 78 additions & 0 deletions prism_benchmarks/data/parser/yjit-bench/2024-07-18-f75db5824.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
{
"results": [
{
"command": "bazel-bin/main/sorbet --parser=sorbet --stop-after=parser ../yjit-bench/benchmarks",
"mean": 13.983836266140003,
"stddev": 0.3753695726271287,
"median": 13.88645245584,
"user": 22.976926319999997,
"system": 0.73527856,
"min": 13.451897627840001,
"max": 14.83432713384,
"times": [
13.451897627840001,
13.78587580584,
13.92110683084,
13.83593312984,
13.85179808084,
14.83432713384,
14.35947390984,
14.01668414084,
13.99022805084,
13.79103795084
],
"exit_codes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"parameters": {
"parser": "sorbet"
}
},
{
"command": "bazel-bin/main/sorbet --parser=prism --stop-after=parser ../yjit-bench/benchmarks",
"mean": 8.63029390654,
"stddev": 0.1018242999685836,
"median": 8.59375904684,
"user": 11.002803519999997,
"system": 0.9380242599999999,
"min": 8.504474396840001,
"max": 8.85641065884,
"times": [
8.57078804984,
8.85641065884,
8.56780114184,
8.69831273484,
8.57807582284,
8.70291158584,
8.60944227084,
8.504474396840001,
8.65632465784,
8.55839774584
],
"exit_codes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"parameters": {
"parser": "prism"
}
}
]
}
78 changes: 78 additions & 0 deletions prism_benchmarks/data/pipeline/2024-07-18-f75db5824.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
{
"results": [
{
"command": "bazel-bin/main/sorbet --parser=sorbet test/prism_regression",
"mean": 0.84178919584,
"stddev": 0.008118653707744158,
"median": 0.83854312314,
"user": 1.59382048,
"system": 0.11926071999999999,
"min": 0.83163896114,
"max": 0.85557230614,
"times": [
0.83945907614,
0.83682301814,
0.85210590814,
0.83473545814,
0.85557230614,
0.83762717014,
0.83645661814,
0.84983627314,
0.84363716914,
0.83163896114
],
"exit_codes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"parameters": {
"parser": "sorbet"
}
},
{
"command": "bazel-bin/main/sorbet --parser=prism test/prism_regression",
"mean": 0.8412279621400002,
"stddev": 0.007882902400181078,
"median": 0.83984855864,
"user": 1.61844638,
"system": 0.10229632,
"min": 0.8312792881400001,
"max": 0.85482773714,
"times": [
0.84042653414,
0.8312792881400001,
0.83306462914,
0.85482773714,
0.85185022614,
0.84023890714,
0.83664591814,
0.8480770171400001,
0.83945821014,
0.83641115414
],
"exit_codes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"parameters": {
"parser": "prism"
}
}
]
}
56 changes: 56 additions & 0 deletions prism_benchmarks/run_prism_benchmarks.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/bin/bash

# This script runs the benchmarks for the Prism parser in the Sorbet type checker.
# It assumes the presence of a few directories and tools:
# - The yjit-bench directory, which contains the benchmarks for YJIT (https://github.com/Shopify/yjit-bench)
# - Hyperfine, a command-line benchmarking tool (https://github.com/sharkdp/hyperfine)

# Check if the yjit-bench directory exists
if [ ! -d "../yjit-bench" ]; then
echo "Please clone the yjit-bench directory before running this script: https://github.com/Shopify/yjit-bench"
exit 1
fi

if [! command -v hyperfine >/dev/null 2>&1 ]; then
echo "Please install hyperfine before running this script: https://github.com/sharkdp/hyperfine"
exit 1
fi

# Enable globstar for recursive directory listing
# This is for Bash 4.0 and later, which does not have recursive globbing enabled by default
shopt -s globstar

# Read all files in the benchmarks directory to warm up the file system cache
for i in {1..10}; do
cat ../yjit-bench/benchmarks/**/*.rb > /dev/null
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was Alex M's idea -- basically warm up the file system cache by reading the benchmark files a few times. I've found that it significantly improves the accuracy of the benchmarks on the first run!

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This step lowering the variance makes me think the # of times hyperfine is warming up may not be enough. It's not crucial but may be worth playing around with different warmup values.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@KaanOzkan agreed. This is significantly faster than warmup loops, so I've done this for now, but I think it's worth reexamining the approach as we go.

cat test/testdata/**/*.rb > /dev/null
done

file_name="$(date '+%Y-%m-%d')-$(git rev-parse --short HEAD).json"

echo "#### Benchmark 1: yjit-bench, parser only"

# Run the parser benchmarks
# These benchmarks compare the performance of the Prism parser with the Sorbet parser without
# any other parts of the type checker enabled.
hyperfine \
--warmup=10 --export-json="prism_benchmarks/data/parser/yjit-bench/$file_name" --parameter-list parser sorbet,prism \
"bazel-bin/main/sorbet --parser={parser} --stop-after=parser ../yjit-bench/benchmarks"

echo "#### Benchmark 2: sorbet-test, parser only"

# Benchmark against sorbet tests to capture performance around parser errors
# Pass the --ignore-failure flag to prevent hyperfine from exiting on parser errors
hyperfine \
--warmup=10 --export-json="prism_benchmarks/data/parser/sorbet-tests/$file_name" --parameter-list parser sorbet,prism \
"bazel-bin/main/sorbet --parser={parser} --stop-after=parser test/testdata" --ignore-failure

echo "#### Benchmark 3: prism regression tests, whole pipeline"

# Run the pipeline benchmarks
# These benchmarks compare the performance of the entire Sorbet type checking pipeline with
# the Prism parser enabled and the Sorbet parser enabled. These benchmarks must be run on a
# smaller set of files that only contain nodes supported by the Prism --> Sorbet translation layer.
hyperfine \
--warmup=10 --export-json="prism_benchmarks/data/pipeline/$file_name" --parameter-list parser sorbet,prism \
"bazel-bin/main/sorbet --parser={parser} test/prism_regression"