Skip to content

Commit 07e57ea

Browse files
tschneidereittschneidereit
authored
Add Web Platform Tests harness (#40)
Since much of our API surface aims to implement existing web specifications like [fetch](https://fetch.spec.whatwg.org/), [Service Workers](https://w3c.github.io/ServiceWorker/), [Streams](https://streams.spec.whatwg.org/), and [encoding](https://encoding.spec.whatwg.org/), the best way to get solid test coverage for our builtins is to run the official test suite for these specifications, [WPT](https://web-platform-tests.org/). This change introduces a custom harness for - running WPT tests - evaluating the results against a set of expectations in a CLI to be used in CI - evaluating the results against a set of expectations in an interactive web interface - updating expectations I think much of this will not be the final word on how to run WPT tests, but it works pretty well for the time being. This change also comes with a list of tests to run (in `tests/wpt-harness/test.json`) and associated expectations (under `tests/wpt-harness/expecations`). These will be added to over time as we improve spec adherence. And finally, this change also modifies the GitHub Actions test workflow to run the WPT harness to ensure that we don't accidentally regress spec adherence.
1 parent f36dee2 commit 07e57ea

File tree

166 files changed

+12896
-12
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

166 files changed

+12896
-12
lines changed

‎.github/workflows/main.yml

+96-10
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,16 @@ on:
88
defaults:
99
run:
1010
shell: bash
11+
env:
12+
wizer_version: 1.3.3
13+
viceroy_version: 0.2.4
1114

1215
jobs:
1316
build_engine:
1417
name: Build Engine
18+
strategy:
19+
matrix:
20+
profile: [debug, release]
1521
runs-on: ubuntu-latest
1622
steps:
1723
- uses: actions/checkout@v2
@@ -23,14 +29,14 @@ jobs:
2329
id: sm-cache
2430
with:
2531
path: |
26-
c-dependencies/spidermonkey/release
27-
key: cache-${{ hashFiles('c-dependencies/spidermonkey/build-engine.sh') }}-${{ hashFiles('c-dependencies/spidermonkey/gecko-revision') }}-${{ hashFiles('c-dependencies/spidermonkey/object-files.list') }}
32+
c-dependencies/spidermonkey/${{ matrix.profile }}
33+
key: cache-${{ hashFiles('c-dependencies/spidermonkey/build-engine.sh') }}-${{ hashFiles('c-dependencies/spidermonkey/gecko-revision') }}-${{ hashFiles('c-dependencies/spidermonkey/object-files.list') }}-${{ matrix.profile }}
2834

2935
- name: "Build SpiderMonkey"
3036
if: steps.sm-cache.outputs.cache-hit != 'true'
3137
run: |
3238
cd c-dependencies/spidermonkey/
33-
bash ./build-engine.sh
39+
bash ./build-engine.sh ${{ matrix.profile }}
3440
3541
- name: "Install wasi-sdk (linux)"
3642
run: |
@@ -50,24 +56,104 @@ jobs:
5056
- name: "Install Binaryen (linux)"
5157
run: |
5258
set -x
53-
export BINARYEN_VERSION=100
59+
export BINARYEN_VERSION=105
5460
curl -sS -L "https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VERSION}/binaryen-version_${BINARYEN_VERSION}-x86_64-linux.tar.gz" | tar xzf - &&
5561
echo "$PWD/binaryen-version_${BINARYEN_VERSION}/bin" >> $GITHUB_PATH
5662
57-
- name: "Build JS runtime"
63+
- name: "Build JS runtime (debug)"
64+
run: |
65+
set -x
66+
mkdir dist
67+
cd dist
68+
DEBUG=1 CXX_OPT="-O1" make -f ../c-dependencies/js-compute-runtime/Makefile
69+
rm *.{d,o}
70+
cd ..
71+
if: matrix.profile == 'debug'
72+
73+
- name: "Build JS runtime (release)"
5874
run: |
5975
set -x
6076
mkdir dist
6177
cd dist
6278
make -f ../c-dependencies/js-compute-runtime/Makefile
6379
rm *.{d,o}
6480
cd ..
81+
if: matrix.profile == 'release'
6582

6683
- uses: actions/upload-artifact@v1
6784
with:
68-
name: engine
85+
name: engine-${{ matrix.profile }}
6986
path: dist
7087

88+
ensure_cargo_installs:
89+
name: Ensure that all required "cargo install" commands are run, or we have a cache hit
90+
strategy:
91+
matrix:
92+
include:
93+
- crate: wizer
94+
version: 1.3.3 # Note: workflow-level env vars can't be used in matrix definitions
95+
options: "--all-features"
96+
- crate: viceroy
97+
version: 0.2.4 # Note: workflow-level env vars can't be used in matrix definitions
98+
options: ""
99+
runs-on: ubuntu-latest
100+
steps:
101+
- name: Cache ${{ matrix.crate }} ${{ matrix.version }}
102+
id: cache-crate
103+
uses: actions/[email protected]
104+
with:
105+
path: "/home/runner/.cargo/bin/${{ matrix.crate }}"
106+
key: crate-cache-${{ matrix.crate }}-${{ matrix.version }}
107+
- name: Install ${{ matrix.crate }} ${{ matrix.version }}
108+
run: cargo install ${{ matrix.crate }} ${{ matrix.options }} --version ${{ matrix.version }}
109+
if: steps.cache-crate.outputs.cache-hit != 'true'
110+
111+
run_wpt:
112+
name: Run Web Platform Tests
113+
strategy:
114+
matrix:
115+
include:
116+
- profile: debug
117+
- profile: release
118+
needs: [build_engine, ensure_cargo_installs]
119+
runs-on: ubuntu-latest
120+
steps:
121+
- uses: actions/checkout@v2
122+
with:
123+
submodules: true
124+
125+
- name: Download Engine
126+
uses: actions/download-artifact@v1
127+
with:
128+
name: engine-${{ matrix.profile }}
129+
130+
- name: Restore Wizer from cache
131+
uses: actions/[email protected]
132+
with:
133+
path: "/home/runner/.cargo/bin/wizer"
134+
key: crate-cache-wizer-${{ env.wizer_version }}
135+
136+
- name: Restore Viceroy from cache
137+
uses: actions/[email protected]
138+
with:
139+
path: "/home/runner/.cargo/bin/viceroy"
140+
key: crate-cache-viceroy-${{ env.viceroy_version }}
141+
142+
- name: Build WPT runtime
143+
run: |
144+
cd engine-${{ matrix.profile }}
145+
bash ../tests/wpt-harness/build-wpt-runtime.sh
146+
147+
- name: Prepare WPT hosts
148+
run: |
149+
cd tests/wpt-harness/wpt
150+
./wpt make-hosts-file | sudo tee -a /etc/hosts
151+
152+
- name: Run tests
153+
run: |
154+
cd engine-${{ matrix.profile }}
155+
node ../tests/wpt-harness/run-wpt.mjs -vv
156+
71157
# Perform release builds of `js-compute-runtime`. Builds on
72158
# Windows/Mac/Linux, and artifacts are uploaded after the build is finished.
73159
# Note that we also run tests here to test exactly what we're deploying.
@@ -90,7 +176,7 @@ jobs:
90176
steps:
91177
- uses: actions/checkout@v2
92178
with:
93-
submodules: true
179+
submodules: false
94180
- uses: ./.github/actions/install-rust
95181
with:
96182
toolchain: ${{ matrix.rust }}
@@ -104,10 +190,10 @@ jobs:
104190
- name: Download Engine
105191
uses: actions/download-artifact@v1
106192
with:
107-
name: engine
193+
name: engine-release
108194

109195
# Build `js-compute-runtime`
110-
- run: PREBUILT_ENGINE=engine/js-compute-runtime.wasm cargo build --release
196+
- run: PREBUILT_ENGINE=engine-release/js-compute-runtime.wasm cargo build --release
111197

112198
# Test what we just built.
113199
# (Disabled for now since we don't actually have any tests for this prototype :/)
@@ -138,7 +224,7 @@ jobs:
138224
# github releases and/or tags for pushes.
139225
publish:
140226
name: Publish
141-
needs: [build]
227+
needs: [build, run_wpt]
142228
runs-on: ubuntu-latest
143229
steps:
144230
- uses: actions/checkout@v2

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
/target
2-
dist
2+
/dist
33
.DS_Store

‎.gitmodules

+5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
11
[submodule "c-dependencies/spidermonkey"]
22
path = c-dependencies/spidermonkey
33
url = https://github.com/tschneidereit/spidermonkey-wasi-embedding.git
4+
shallow = true
5+
[submodule "tests/wpt-harness/wpt"]
6+
path = tests/wpt-harness/wpt
7+
url = https://github.com/web-platform-tests/wpt.git
8+
shallow = true

‎.vscode/settings.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -74,5 +74,6 @@
7474
"typeinfo": "cpp",
7575
"unordered_map": "cpp",
7676
"utility": "cpp"
77-
}
77+
},
78+
"git.ignoreLimitWarning": true
7879
}

‎tests/wpt-harness/build-wpt-runtime.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -e
3+
4+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
5+
6+
cat $script_dir/pre-harness.js $script_dir/wpt/resources/testharness.js $script_dir/post-harness.js | wizer --allow-wasi --dir=. -r _start=wizer.resume -o wpt-runtime.wasm js-compute-runtime.wasm

‎tests/wpt-harness/expectations/WebCryptoAPI/getRandomValues.any.js.json

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{
2+
"Float arrays": {
3+
"status": 1
4+
},
5+
"Integer array: Int8Array": {
6+
"status": 0
7+
},
8+
"Large length: Int8Array": {
9+
"status": 1
10+
},
11+
"Null arrays: Int8Array": {
12+
"status": 0
13+
},
14+
"Integer array: Int16Array": {
15+
"status": 0
16+
},
17+
"Large length: Int16Array": {
18+
"status": 1
19+
},
20+
"Null arrays: Int16Array": {
21+
"status": 0
22+
},
23+
"Integer array: Int32Array": {
24+
"status": 0
25+
},
26+
"Large length: Int32Array": {
27+
"status": 1
28+
},
29+
"Null arrays: Int32Array": {
30+
"status": 0
31+
},
32+
"Integer array: BigInt64Array": {
33+
"status": 0
34+
},
35+
"Large length: BigInt64Array": {
36+
"status": 1
37+
},
38+
"Null arrays: BigInt64Array": {
39+
"status": 0
40+
},
41+
"Integer array: Uint8Array": {
42+
"status": 0
43+
},
44+
"Large length: Uint8Array": {
45+
"status": 1
46+
},
47+
"Null arrays: Uint8Array": {
48+
"status": 0
49+
},
50+
"Integer array: Uint8ClampedArray": {
51+
"status": 0
52+
},
53+
"Large length: Uint8ClampedArray": {
54+
"status": 1
55+
},
56+
"Null arrays: Uint8ClampedArray": {
57+
"status": 0
58+
},
59+
"Integer array: Uint16Array": {
60+
"status": 0
61+
},
62+
"Large length: Uint16Array": {
63+
"status": 1
64+
},
65+
"Null arrays: Uint16Array": {
66+
"status": 0
67+
},
68+
"Integer array: Uint32Array": {
69+
"status": 0
70+
},
71+
"Large length: Uint32Array": {
72+
"status": 1
73+
},
74+
"Null arrays: Uint32Array": {
75+
"status": 0
76+
},
77+
"Integer array: BigUint64Array": {
78+
"status": 0
79+
},
80+
"Large length: BigUint64Array": {
81+
"status": 1
82+
},
83+
"Null arrays: BigUint64Array": {
84+
"status": 0
85+
}
86+
}

‎tests/wpt-harness/expectations/encoding/api-basics.any.js.json

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"Default encodings": {
3+
"status": 0
4+
},
5+
"Default inputs": {
6+
"status": 1
7+
},
8+
"Encode/decode round trip: utf-8": {
9+
"status": 0
10+
},
11+
"Decode sample: utf-16le": {
12+
"status": 1
13+
},
14+
"Decode sample: utf-16be": {
15+
"status": 1
16+
},
17+
"Decode sample: utf-16": {
18+
"status": 1
19+
}
20+
}

‎tests/wpt-harness/expectations/encoding/api-invalid-label.any.js.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

‎tests/wpt-harness/expectations/encoding/api-surrogates-utf8.any.js.json

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"Invalid surrogates encoded into UTF-8: Sanity check": {
3+
"status": 0
4+
},
5+
"Invalid surrogates encoded into UTF-8: Surrogate half (low)": {
6+
"status": 0
7+
},
8+
"Invalid surrogates encoded into UTF-8: Surrogate half (high)": {
9+
"status": 0
10+
},
11+
"Invalid surrogates encoded into UTF-8: Surrogate half (low), in a string": {
12+
"status": 0
13+
},
14+
"Invalid surrogates encoded into UTF-8: Surrogate half (high), in a string": {
15+
"status": 0
16+
},
17+
"Invalid surrogates encoded into UTF-8: Wrong order": {
18+
"status": 0
19+
}
20+
}

0 commit comments

Comments
 (0)